org.apache.flume.ChannelSelector - java examples

Here are the examples of the java api org.apache.flume.ChannelSelector taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

82 Examples 7

19 View Complete Implementation : RedisSourceTest.java
Copyright Apache License 2.0
Author : Stratio
@Before
public void setUp() {
    source = new RedisSource();
    channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
}

19 View Complete Implementation : TestChannelProcessor.java
Copyright Apache License 2.0
Author : javachen
/**
 * Ensure that we see the original NPE from the PreConditions check instead
 * of an auto-generated NPE, which could be masking something else.
 */
@Test
public void testNullFromGetTransaction() {
    // channel which returns null from getTransaction()
    Channel ch = mock(Channel.clreplaced);
    when(ch.getTransaction()).thenReturn(null);
    ChannelSelector sel = new ReplicatingChannelSelector();
    sel.setChannels(Lists.newArrayList(ch));
    ChannelProcessor proc = new ChannelProcessor(sel);
    List<Event> events = Lists.newArrayList();
    events.add(EventBuilder.withBody("event 1", Charsets.UTF_8));
    boolean threw = false;
    try {
        proc.processEventBatch(events);
    } catch (NullPointerException ex) {
        threw = true;
        replacedert.replacedertNotNull("NPE must be manually thrown", ex.getMessage());
    }
    replacedert.replacedertTrue("Must throw NPE", threw);
}

19 View Complete Implementation : SNMPSourceTestIT.java
Copyright Apache License 2.0
Author : Stratio
@Before
public void setUp() {
    source = new SNMPSource();
    channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
}

18 View Complete Implementation : TestChannelProcessor.java
Copyright Apache License 2.0
Author : javachen
/**
 * Ensure that we bubble up any specific exception thrown from getTransaction
 * instead of another exception masking it such as an NPE
 */
@Test(expected = ChannelException.clreplaced)
public void testExceptionFromGetTransaction() {
    // create a channel which unexpectedly throws a ChEx on getTransaction()
    Channel ch = mock(Channel.clreplaced);
    when(ch.getTransaction()).thenThrow(new ChannelException("doh!"));
    ChannelSelector sel = new ReplicatingChannelSelector();
    sel.setChannels(Lists.newArrayList(ch));
    ChannelProcessor proc = new ChannelProcessor(sel);
    List<Event> events = Lists.newArrayList();
    events.add(EventBuilder.withBody("event 1", Charsets.UTF_8));
    proc.processEventBatch(events);
}

18 View Complete Implementation : TestReplicatingChannelSelector.java
Copyright Apache License 2.0
Author : cloudera
public clreplaced TestReplicatingChannelSelector {

    private List<Channel> channels = new ArrayList<Channel>();

    private ChannelSelector selector;

    @Before
    public void setUp() throws Exception {
        channels.clear();
        channels.add(MockChannel.createMockChannel("ch1"));
        channels.add(MockChannel.createMockChannel("ch2"));
        channels.add(MockChannel.createMockChannel("ch3"));
        selector = ChannelSelectorFactory.create(channels, new HashMap<String, String>());
    }

    @Test
    public void testReplicatingSelector() throws Exception {
        List<Channel> channels = selector.getRequiredChannels(new MockEvent());
        replacedert.replacedertNotNull(channels);
        replacedert.replacedertTrue(channels.size() == 3);
        replacedert.replacedertTrue(channels.get(0).getName().equals("ch1"));
        replacedert.replacedertTrue(channels.get(1).getName().equals("ch2"));
        replacedert.replacedertTrue(channels.get(2).getName().equals("ch3"));
        List<Channel> optCh = selector.getOptionalChannels(new MockEvent());
        replacedert.replacedertTrue(optCh.size() == 0);
    }
}

18 View Complete Implementation : TestAvroSource.java
Copyright Apache License 2.0
Author : apache
@Before
public void setUp() throws UnknownHostException {
    localhost = InetAddress.getByName("127.0.0.1");
    source = new AvroSource();
    channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
}

18 View Complete Implementation : ChannelSelectorFactory.java
Copyright Apache License 2.0
Author : apache
private static ChannelSelector getSelectorForType(String type) {
    if (type == null || type.trim().length() == 0) {
        return new ReplicatingChannelSelector();
    }
    String selectorClreplacedName = type;
    ChannelSelectorType selectorType = ChannelSelectorType.OTHER;
    try {
        selectorType = ChannelSelectorType.valueOf(type.toUpperCase(Locale.ENGLISH));
    } catch (IllegalArgumentException ex) {
        LOGGER.debug("Selector type {} is a custom type", type);
    }
    if (!selectorType.equals(ChannelSelectorType.OTHER)) {
        selectorClreplacedName = selectorType.getChannelSelectorClreplacedName();
    }
    ChannelSelector selector = null;
    try {
        @SuppressWarnings("unchecked")
        Clreplaced<? extends ChannelSelector> selectorClreplaced = (Clreplaced<? extends ChannelSelector>) Clreplaced.forName(selectorClreplacedName);
        selector = selectorClreplaced.newInstance();
    } catch (Exception ex) {
        throw new FlumeException("Unable to load selector type: " + type + ", clreplaced: " + selectorClreplacedName, ex);
    }
    return selector;
}

18 View Complete Implementation : TestExecSource.java
Copyright Apache License 2.0
Author : apache
public clreplaced TestExecSource {

    private AbstractSource source;

    private Channel channel = new MemoryChannel();

    private Context context = new Context();

    private ChannelSelector rcs = new ReplicatingChannelSelector();

    @Before
    public void setUp() {
        context.put("keep-alive", "1");
        context.put("capacity", "1000");
        context.put("transactionCapacity", "1000");
        Configurables.configure(channel, context);
        rcs.setChannels(Lists.newArrayList(channel));
        source = new ExecSource();
        source.setChannelProcessor(new ChannelProcessor(rcs));
    }

    @After
    public void tearDown() {
        source.stop();
        // Remove the MBean registered for Monitoring
        ObjectName objName = null;
        try {
            objName = new ObjectName("org.apache.flume.source" + ":type=" + source.getName());
            ManagementFactory.getPlatformMBeanServer().unregisterMBean(objName);
        } catch (Exception ex) {
            System.out.println("Failed to unregister the monitored counter: " + objName + ex.getMessage());
        }
    }

    @Test
    public void testProcess() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        // Generates a random files for input\output
        File inputFile = File.createTempFile("input", null);
        File ouputFile = File.createTempFile("ouput", null);
        FileUtils.forceDeleteOnExit(inputFile);
        FileUtils.forceDeleteOnExit(ouputFile);
        // Generates input file with a random data set (10 lines, 200 characters each)
        FileOutputStream outputStream1 = new FileOutputStream(inputFile);
        for (int i = 0; i < 10; i++) {
            outputStream1.write(RandomStringUtils.randomAlphanumeric(200).getBytes());
            outputStream1.write('\n');
        }
        outputStream1.close();
        String command = SystemUtils.IS_OS_WINDOWS ? String.format("cmd /c type %s", inputFile.getAbsolutePath()) : String.format("cat %s", inputFile.getAbsolutePath());
        context.put("command", command);
        context.put("keep-alive", "1");
        context.put("capacity", "1000");
        context.put("transactionCapacity", "1000");
        Configurables.configure(source, context);
        source.start();
        Thread.sleep(2000);
        Transaction transaction = channel.getTransaction();
        transaction.begin();
        Event event;
        FileOutputStream outputStream = new FileOutputStream(ouputFile);
        while ((event = channel.take()) != null) {
            outputStream.write(event.getBody());
            outputStream.write('\n');
        }
        outputStream.close();
        transaction.commit();
        transaction.close();
        replacedert.replacedertEquals(FileUtils.checksumCRC32(inputFile), FileUtils.checksumCRC32(ouputFile));
    }

    @Test
    public void testShellCommandSimple() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        if (SystemUtils.IS_OS_WINDOWS) {
            runTestShellCmdHelper("powershell -ExecutionPolicy Unrestricted -command", "1..5", new String[] { "1", "2", "3", "4", "5" });
        } else {
            runTestShellCmdHelper("/bin/bash -c", "seq 5", new String[] { "1", "2", "3", "4", "5" });
        }
    }

    @Test
    public void testShellCommandBackTicks() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        // command with backticks
        if (SystemUtils.IS_OS_WINDOWS) {
            runTestShellCmdHelper("powershell -ExecutionPolicy Unrestricted -command", "$(1..5)", new String[] { "1", "2", "3", "4", "5" });
        } else {
            runTestShellCmdHelper("/bin/bash -c", "echo `seq 5`", new String[] { "1 2 3 4 5" });
            runTestShellCmdHelper("/bin/bash -c", "echo $(seq 5)", new String[] { "1 2 3 4 5" });
        }
    }

    @Test
    public void testShellCommandComplex() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        // command with wildcards & pipes
        String[] expected = { "1234", "abcd", "ijk", "xyz", "zzz" };
        // pipes
        if (SystemUtils.IS_OS_WINDOWS) {
            runTestShellCmdHelper("powershell -ExecutionPolicy Unrestricted -command", "'zzz','1234','xyz','abcd','ijk' | sort", expected);
        } else {
            runTestShellCmdHelper("/bin/bash -c", "echo zzz 1234 xyz abcd ijk | xargs -n1 echo | sort -f", expected);
        }
    }

    @Test
    public void testShellCommandScript() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        // mini script
        if (SystemUtils.IS_OS_WINDOWS) {
            runTestShellCmdHelper("powershell -ExecutionPolicy Unrestricted -command", "foreach ($i in 1..5) { $i }", new String[] { "1", "2", "3", "4", "5" });
            // shell arithmetic
            runTestShellCmdHelper("powershell -ExecutionPolicy Unrestricted -command", "if(2+2 -gt 3) { 'good' } else { 'not good' } ", new String[] { "good" });
        } else {
            runTestShellCmdHelper("/bin/bash -c", "for i in {1..5}; do echo $i;done", new String[] { "1", "2", "3", "4", "5" });
            // shell arithmetic
            runTestShellCmdHelper("/bin/bash -c", "if ((2+2>3)); " + "then  echo good; else echo not good; fi", new String[] { "good" });
        }
    }

    @Test
    public void testShellCommandEmbeddingAndEscaping() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        // mini script
        String fileName = SystemUtils.IS_OS_WINDOWS ? "src\\test\\resources\\test_command.ps1" : "src/test/resources/test_command.txt";
        BufferedReader reader = new BufferedReader(new FileReader(fileName));
        try {
            String shell = SystemUtils.IS_OS_WINDOWS ? "powershell -ExecutionPolicy Unrestricted -command" : "/bin/bash -c";
            String command1 = reader.readLine();
            replacedert.replacedertNotNull(command1);
            String[] output1 = new String[] { "'1'", "\"2\"", "\\3", "\\4" };
            runTestShellCmdHelper(shell, command1, output1);
            String command2 = reader.readLine();
            replacedert.replacedertNotNull(command2);
            String[] output2 = new String[] { "1", "2", "3", "4", "5" };
            runTestShellCmdHelper(shell, command2, output2);
            String command3 = reader.readLine();
            replacedert.replacedertNotNull(command3);
            String[] output3 = new String[] { "2", "3", "4", "5", "6" };
            runTestShellCmdHelper(shell, command3, output3);
        } finally {
            reader.close();
        }
    }

    @Test
    public void testMonitoredCounterGroup() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        // mini script
        if (SystemUtils.IS_OS_WINDOWS) {
            runTestShellCmdHelper("powershell -ExecutionPolicy Unrestricted -command", "foreach ($i in 1..5) { $i }", new String[] { "1", "2", "3", "4", "5" });
        } else {
            runTestShellCmdHelper("/bin/bash -c", "for i in {1..5}; do echo $i;done", new String[] { "1", "2", "3", "4", "5" });
        }
        ObjectName objName = null;
        try {
            objName = new ObjectName("org.apache.flume.source" + ":type=" + source.getName());
            MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
            String[] strAtts = { "Type", "EventReceivedCount", "EventAcceptedCount" };
            AttributeList attrList = mbeanServer.getAttributes(objName, strAtts);
            replacedert.replacedertNotNull(attrList.get(0));
            replacedert.replacedertEquals("Expected Value: Type", "Type", ((Attribute) attrList.get(0)).getName());
            replacedert.replacedertEquals("Expected Value: SOURCE", "SOURCE", ((Attribute) attrList.get(0)).getValue());
            replacedert.replacedertNotNull(attrList.get(1));
            replacedert.replacedertEquals("Expected Value: EventReceivedCount", "EventReceivedCount", ((Attribute) attrList.get(1)).getName());
            replacedert.replacedertEquals("Expected Value: 5", "5", ((Attribute) attrList.get(1)).getValue().toString());
            replacedert.replacedertNotNull(attrList.get(2));
            replacedert.replacedertEquals("Expected Value: EventAcceptedCount", "EventAcceptedCount", ((Attribute) attrList.get(2)).getName());
            replacedert.replacedertEquals("Expected Value: 5", "5", ((Attribute) attrList.get(2)).getValue().toString());
        } catch (Exception ex) {
            System.out.println("Unable to retreive the monitored counter: " + objName + ex.getMessage());
        }
    }

    @Test
    public void testBatchTimeout() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        String filePath = "/tmp/flume-execsource." + Thread.currentThread().getId();
        String eventBody = "TestMessage";
        FileOutputStream outputStream = new FileOutputStream(filePath);
        context.put(ExecSourceConfigurationConstants.CONFIG_BATCH_SIZE, "50000");
        context.put(ExecSourceConfigurationConstants.CONFIG_BATCH_TIME_OUT, "750");
        context.put("shell", SystemUtils.IS_OS_WINDOWS ? "powershell -ExecutionPolicy Unrestricted -command" : "/bin/bash -c");
        context.put("command", SystemUtils.IS_OS_WINDOWS ? "Get-Content " + filePath + " | Select-Object -Last 10" : ("tail -f " + filePath));
        Configurables.configure(source, context);
        source.start();
        Transaction transaction = channel.getTransaction();
        transaction.begin();
        for (int lineNumber = 0; lineNumber < 3; lineNumber++) {
            outputStream.write((eventBody).getBytes());
            outputStream.write(String.valueOf(lineNumber).getBytes());
            outputStream.write('\n');
            outputStream.flush();
        }
        outputStream.close();
        Thread.sleep(1500);
        for (int i = 0; i < 3; i++) {
            Event event = channel.take();
            replacedertNotNull(event);
            replacedertNotNull(event.getBody());
            replacedertEquals(eventBody + String.valueOf(i), new String(event.getBody()));
        }
        transaction.commit();
        transaction.close();
        source.stop();
        File file = new File(filePath);
        FileUtils.forceDelete(file);
    }

    private void runTestShellCmdHelper(String shell, String command, String[] expectedOutput) throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        context.put("shell", shell);
        context.put("command", command);
        Configurables.configure(source, context);
        source.start();
        // Some commands might take longer to complete, specially on Windows
        // or on slow environments (e.g. Travis CI).
        Thread.sleep(2500);
        Transaction transaction = channel.getTransaction();
        transaction.begin();
        try {
            List<String> output = Lists.newArrayList();
            Event event;
            while ((event = channel.take()) != null) {
                output.add(new String(event.getBody(), Charset.defaultCharset()));
            }
            transaction.commit();
            replacedert.replacedertArrayEquals(expectedOutput, output.toArray(new String[] {}));
        } finally {
            transaction.close();
            source.stop();
        }
    }

    @Test
    public void testRestart() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        context.put(ExecSourceConfigurationConstants.CONFIG_RESTART_THROTTLE, "10");
        context.put(ExecSourceConfigurationConstants.CONFIG_RESTART, "true");
        context.put("command", SystemUtils.IS_OS_WINDOWS ? "cmd /c echo flume" : "echo flume");
        Configurables.configure(source, context);
        source.start();
        Transaction transaction = channel.getTransaction();
        transaction.begin();
        long start = System.currentTimeMillis();
        for (int i = 0; i < 5; i++) {
            Event event = channel.take();
            replacedertNotNull(event);
            replacedertNotNull(event.getBody());
            replacedertEquals("flume", new String(event.getBody(), Charsets.UTF_8));
        }
        // ensure restartThrottle was turned down as expected
        replacedertTrue(System.currentTimeMillis() - start < 10000L);
        transaction.commit();
        transaction.close();
        source.stop();
    }

    /**
     * Tests to make sure that the shutdown mechanism works. There are races
     * in this test if the system has another sleep command running with the
     * same sleep interval but we pick rarely used sleep times and make an
     * effort to detect if our sleep time is already in use. Note the
     * ps -ef command should work on both macs and linux.
     */
    @Test
    public void testShutdown() throws Exception {
        // pick a rare sleep time
        int seconds = 272;
        // now find one that is not in use
        boolean searchForCommand = true;
        while (searchForCommand) {
            searchForCommand = false;
            String command = SystemUtils.IS_OS_WINDOWS ? "cmd /c sleep " + seconds : "sleep " + seconds;
            String searchTxt = SystemUtils.IS_OS_WINDOWS ? "sleep.exe" : "\b" + command + "\b";
            Pattern pattern = Pattern.compile(searchTxt);
            for (String line : exec(SystemUtils.IS_OS_WINDOWS ? "cmd /c tasklist /FI \"SESSIONNAME eq Console\"" : "ps -ef")) {
                if (pattern.matcher(line).find()) {
                    seconds++;
                    searchForCommand = true;
                    break;
                }
            }
        }
        // yes in the mean time someone could use our sleep time
        // but this should be a fairly rare scenario
        String command = "sleep " + seconds;
        Pattern pattern = Pattern.compile("\b" + command + "\b");
        context.put(ExecSourceConfigurationConstants.CONFIG_RESTART, "false");
        context.put("command", command);
        Configurables.configure(source, context);
        source.start();
        Thread.sleep(1000L);
        source.stop();
        Thread.sleep(1000L);
        for (String line : exec(SystemUtils.IS_OS_WINDOWS ? "cmd /c tasklist /FI \"SESSIONNAME eq Console\"" : "ps -ef")) {
            if (pattern.matcher(line).find()) {
                replacedert.fail("Found [" + line + "]");
            }
        }
    }

    private static List<String> exec(String command) throws Exception {
        String[] commandArgs = command.split("\\s+");
        Process process = new ProcessBuilder(commandArgs).start();
        BufferedReader reader = null;
        try {
            reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
            List<String> result = Lists.newArrayList();
            String line;
            while ((line = reader.readLine()) != null) {
                result.add(line);
            }
            return result;
        } finally {
            process.destroy();
            if (reader != null) {
                reader.close();
            }
            int exit = process.waitFor();
            if (exit != 0) {
                throw new IllegalStateException("Command [" + command + "] exited with " + exit);
            }
        }
    }
}

18 View Complete Implementation : ChannelSelectorFactory.java
Copyright Apache License 2.0
Author : apache
public static ChannelSelector create(List<Channel> channels, ChannelSelectorConfiguration conf) {
    String type = ChannelSelectorType.REPLICATING.toString();
    if (conf != null) {
        type = conf.getType();
    }
    ChannelSelector selector = getSelectorForType(type);
    selector.setChannels(channels);
    Configurables.configure(selector, conf);
    return selector;
}

18 View Complete Implementation : TestNetcatUdpSource.java
Copyright Apache License 2.0
Author : apache
private void init() {
    source = new NetcatUdpSource();
    channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
    Context context = new Context();
    context.put("port", String.valueOf(TEST_NETCAT_PORT));
    source.configure(context);
}

18 View Complete Implementation : TestSpoolDirectorySource.java
Copyright Apache License 2.0
Author : apache
@Before
public void setUp() {
    source = new SpoolDirectorySource();
    channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
    tmpDir = Files.createTempDir();
}

18 View Complete Implementation : TestMultiplexingChannelSelector.java
Copyright Apache License 2.0
Author : cloudera
public clreplaced TestMultiplexingChannelSelector {

    private List<Channel> channels = new ArrayList<Channel>();

    private ChannelSelector selector;

    @Before
    public void setUp() throws Exception {
        channels.clear();
        channels.add(MockChannel.createMockChannel("ch1"));
        channels.add(MockChannel.createMockChannel("ch2"));
        channels.add(MockChannel.createMockChannel("ch3"));
        Map<String, String> config = new HashMap<String, String>();
        config.put("type", "multiplexing");
        config.put("header", "myheader");
        config.put("mapping.foo", "ch1 ch2");
        config.put("mapping.bar", "ch2 ch3");
        config.put("mapping.xyz", "ch1 ch2 ch3");
        config.put("default", "ch1 ch3");
        selector = ChannelSelectorFactory.create(channels, config);
    }

    @Test
    public void testSelection() throws Exception {
        replacedert.replacedertTrue(selector instanceof MultiplexingChannelSelector);
        Event event1 = new MockEvent();
        Map<String, String> header1 = new HashMap<String, String>();
        // should match ch1 ch2
        header1.put("myheader", "foo");
        event1.setHeaders(header1);
        List<Channel> reqCh1 = selector.getRequiredChannels(event1);
        replacedert.replacedertEquals(2, reqCh1.size());
        replacedert.replacedertTrue(reqCh1.get(0).getName().equals("ch1"));
        replacedert.replacedertTrue(reqCh1.get(1).getName().equals("ch2"));
        List<Channel> optCh1 = selector.getOptionalChannels(event1);
        replacedert.replacedertTrue(optCh1.size() == 0);
        Event event2 = new MockEvent();
        Map<String, String> header2 = new HashMap<String, String>();
        // should match ch2 ch3
        header2.put("myheader", "bar");
        event2.setHeaders(header2);
        List<Channel> reqCh2 = selector.getRequiredChannels(event2);
        replacedert.replacedertEquals(2, reqCh2.size());
        replacedert.replacedertTrue(reqCh2.get(0).getName().equals("ch2"));
        replacedert.replacedertTrue(reqCh2.get(1).getName().equals("ch3"));
        List<Channel> optCh2 = selector.getOptionalChannels(event2);
        replacedert.replacedertTrue(optCh2.size() == 0);
        Event event3 = new MockEvent();
        Map<String, String> header3 = new HashMap<String, String>();
        // should match ch1 ch2 ch3
        header3.put("myheader", "xyz");
        event3.setHeaders(header3);
        List<Channel> reqCh3 = selector.getRequiredChannels(event3);
        replacedert.replacedertEquals(3, reqCh3.size());
        replacedert.replacedertTrue(reqCh3.get(0).getName().equals("ch1"));
        replacedert.replacedertTrue(reqCh3.get(1).getName().equals("ch2"));
        replacedert.replacedertTrue(reqCh3.get(2).getName().equals("ch3"));
        List<Channel> optCh3 = selector.getOptionalChannels(event3);
        replacedert.replacedertTrue(optCh3.size() == 0);
    }

    // If the header information cannot map the event to any of the channels
    // it should always be mapped to the default channel(s).
    @Test
    public void testNoSelection() throws Exception {
        replacedert.replacedertTrue(selector instanceof MultiplexingChannelSelector);
        Event noHeaderEvent = new MockEvent();
        List<Channel> reqCh1 = selector.getRequiredChannels(noHeaderEvent);
        List<Channel> optCh1 = selector.getOptionalChannels(noHeaderEvent);
        replacedert.replacedertEquals(2, reqCh1.size());
        replacedert.replacedertTrue(reqCh1.get(0).getName().equals("ch1"));
        replacedert.replacedertTrue(reqCh1.get(1).getName().equals("ch3"));
        replacedert.replacedertTrue(optCh1.size() == 0);
        Map<String, String> header2 = new HashMap<String, String>();
        header2.put("someheader", "foo");
        Event invalidHeaderEvent = new MockEvent();
        invalidHeaderEvent.setHeaders(header2);
        List<Channel> reqCh2 = selector.getRequiredChannels(invalidHeaderEvent);
        List<Channel> optCh2 = selector.getOptionalChannels(invalidHeaderEvent);
        replacedert.replacedertEquals(2, reqCh2.size());
        replacedert.replacedertTrue(reqCh2.get(0).getName().equals("ch1"));
        replacedert.replacedertTrue(reqCh2.get(1).getName().equals("ch3"));
        replacedert.replacedertTrue(optCh2.size() == 0);
        Map<String, String> header3 = new HashMap<String, String>();
        header3.put("myheader", "bar1");
        Event unmatchedHeaderEvent = new MockEvent();
        unmatchedHeaderEvent.setHeaders(header3);
        List<Channel> reqCh3 = selector.getRequiredChannels(unmatchedHeaderEvent);
        List<Channel> optCh3 = selector.getOptionalChannels(unmatchedHeaderEvent);
        replacedert.replacedertEquals(2, reqCh3.size());
        replacedert.replacedertTrue(reqCh3.get(0).getName().equals("ch1"));
        replacedert.replacedertTrue(reqCh3.get(1).getName().equals("ch3"));
        replacedert.replacedertTrue(optCh3.size() == 0);
        List<Channel> allChannels = selector.getAllChannels();
        replacedert.replacedertTrue(allChannels.size() == 3);
        replacedert.replacedertTrue(allChannels.get(0).getName().equals("ch1"));
        replacedert.replacedertTrue(allChannels.get(1).getName().equals("ch2"));
        replacedert.replacedertTrue(allChannels.get(2).getName().equals("ch3"));
    }
}

18 View Complete Implementation : TestMultiplexingChannelSelector.java
Copyright Apache License 2.0
Author : apache
public clreplaced TestMultiplexingChannelSelector {

    private List<Channel> channels = new ArrayList<Channel>();

    private ChannelSelector selector;

    private Map<String, String> config = new HashMap<String, String>();

    @Before
    public void setUp() throws Exception {
        channels.clear();
        channels.add(MockChannel.createMockChannel("ch1"));
        channels.add(MockChannel.createMockChannel("ch2"));
        channels.add(MockChannel.createMockChannel("ch3"));
        config.put("type", "multiplexing");
        config.put("header", "myheader");
        config.put("optional.foo", "ch2 ch3");
        config.put("optional.xyz", "ch1 ch3");
        config.put("optional.zebra", "ch1 ch2");
    }

    @Test
    public void testSelection() throws Exception {
        config.put("mapping.foo", "ch1 ch2");
        config.put("mapping.bar", "ch2 ch3");
        config.put("mapping.xyz", "ch1 ch2 ch3");
        config.put("default", "ch1 ch3");
        selector = ChannelSelectorFactory.create(channels, config);
        replacedert.replacedertTrue(selector instanceof MultiplexingChannelSelector);
        Event event1 = new MockEvent();
        Map<String, String> header1 = new HashMap<String, String>();
        // should match ch1 ch2
        header1.put("myheader", "foo");
        event1.setHeaders(header1);
        List<Channel> reqCh1 = selector.getRequiredChannels(event1);
        replacedert.replacedertEquals(2, reqCh1.size());
        replacedert.replacedertTrue(reqCh1.get(0).getName().equals("ch1"));
        replacedert.replacedertTrue(reqCh1.get(1).getName().equals("ch2"));
        List<Channel> optCh1 = selector.getOptionalChannels(event1);
        replacedert.replacedertTrue(optCh1.size() == 1);
        // ch2 should not be there -- since it is a required channel
        replacedert.replacedertTrue(optCh1.get(0).getName().equals("ch3"));
        Event event2 = new MockEvent();
        Map<String, String> header2 = new HashMap<String, String>();
        // should match ch2 ch3
        header2.put("myheader", "bar");
        event2.setHeaders(header2);
        List<Channel> reqCh2 = selector.getRequiredChannels(event2);
        replacedert.replacedertEquals(2, reqCh2.size());
        replacedert.replacedertTrue(reqCh2.get(0).getName().equals("ch2"));
        replacedert.replacedertTrue(reqCh2.get(1).getName().equals("ch3"));
        List<Channel> optCh2 = selector.getOptionalChannels(event2);
        replacedert.replacedertTrue(optCh2.isEmpty());
        Event event3 = new MockEvent();
        Map<String, String> header3 = new HashMap<String, String>();
        // should match ch1 ch2 ch3
        header3.put("myheader", "xyz");
        event3.setHeaders(header3);
        List<Channel> reqCh3 = selector.getRequiredChannels(event3);
        replacedert.replacedertEquals(3, reqCh3.size());
        replacedert.replacedertTrue(reqCh3.get(0).getName().equals("ch1"));
        replacedert.replacedertTrue(reqCh3.get(1).getName().equals("ch2"));
        replacedert.replacedertTrue(reqCh3.get(2).getName().equals("ch3"));
        List<Channel> optCh3 = selector.getOptionalChannels(event3);
        // All of the optional channels should go away.
        replacedert.replacedertTrue(optCh3.size() == 0);
    }

    // If the header information cannot map the event to any of the channels
    // it should always be mapped to the default channel(s).
    @Test
    public void testNoSelection() throws Exception {
        config.put("mapping.foo", "ch1 ch2");
        config.put("mapping.bar", "ch2 ch3");
        config.put("mapping.xyz", "ch1 ch2 ch3");
        config.put("default", "ch1 ch3");
        selector = ChannelSelectorFactory.create(channels, config);
        replacedert.replacedertTrue(selector instanceof MultiplexingChannelSelector);
        Event noHeaderEvent = new MockEvent();
        List<Channel> reqCh1 = selector.getRequiredChannels(noHeaderEvent);
        List<Channel> optCh1 = selector.getOptionalChannels(noHeaderEvent);
        replacedert.replacedertEquals(2, reqCh1.size());
        replacedert.replacedertTrue(reqCh1.get(0).getName().equals("ch1"));
        replacedert.replacedertTrue(reqCh1.get(1).getName().equals("ch3"));
        replacedert.replacedertTrue(optCh1.isEmpty());
        Map<String, String> header2 = new HashMap<String, String>();
        header2.put("someheader", "foo");
        Event invalidHeaderEvent = new MockEvent();
        invalidHeaderEvent.setHeaders(header2);
        List<Channel> reqCh2 = selector.getRequiredChannels(invalidHeaderEvent);
        List<Channel> optCh2 = selector.getOptionalChannels(invalidHeaderEvent);
        replacedert.replacedertEquals(2, reqCh2.size());
        replacedert.replacedertTrue(reqCh2.get(0).getName().equals("ch1"));
        replacedert.replacedertTrue(reqCh2.get(1).getName().equals("ch3"));
        replacedert.replacedertTrue(optCh2.isEmpty());
        Map<String, String> header3 = new HashMap<String, String>();
        header3.put("myheader", "bar1");
        Event unmatchedHeaderEvent = new MockEvent();
        unmatchedHeaderEvent.setHeaders(header3);
        List<Channel> reqCh3 = selector.getRequiredChannels(unmatchedHeaderEvent);
        List<Channel> optCh3 = selector.getOptionalChannels(unmatchedHeaderEvent);
        replacedert.replacedertEquals(2, reqCh3.size());
        replacedert.replacedertTrue(reqCh3.get(0).getName().equals("ch1"));
        replacedert.replacedertTrue(reqCh3.get(1).getName().equals("ch3"));
        replacedert.replacedertTrue(optCh3.isEmpty());
        Map<String, String> header4 = new HashMap<String, String>();
        header4.put("myheader", "zebra");
        Event zebraEvent = new MockEvent();
        zebraEvent.setHeaders(header4);
        List<Channel> reqCh4 = selector.getRequiredChannels(zebraEvent);
        List<Channel> optCh4 = selector.getOptionalChannels(zebraEvent);
        replacedert.replacedertEquals(2, reqCh4.size());
        replacedert.replacedertTrue(reqCh4.get(0).getName().equals("ch1"));
        replacedert.replacedertTrue(reqCh4.get(1).getName().equals("ch3"));
        // Since ch1 is also in default list, it is removed.
        replacedert.replacedertTrue(optCh4.size() == 1);
        replacedert.replacedertTrue(optCh4.get(0).getName().equals("ch2"));
        List<Channel> allChannels = selector.getAllChannels();
        replacedert.replacedertTrue(allChannels.size() == 3);
        replacedert.replacedertTrue(allChannels.get(0).getName().equals("ch1"));
        replacedert.replacedertTrue(allChannels.get(1).getName().equals("ch2"));
        replacedert.replacedertTrue(allChannels.get(2).getName().equals("ch3"));
    }

    @Test
    public void testNoDefault() {
        config.put("mapping.foo", "ch1 ch2");
        config.put("mapping.bar", "ch2 ch3");
        config.put("mapping.xyz", "ch1 ch2 ch3");
        config.put("mapping.zebra", "ch2");
        config.put("optional.zebra", "ch1 ch3");
        selector = ChannelSelectorFactory.create(channels, config);
        replacedert.replacedertTrue(selector instanceof MultiplexingChannelSelector);
        Event event1 = new MockEvent();
        Map<String, String> header1 = new HashMap<String, String>();
        // should match ch1 ch2
        header1.put("myheader", "foo");
        event1.setHeaders(header1);
        List<Channel> reqCh1 = selector.getRequiredChannels(event1);
        replacedert.replacedertEquals(2, reqCh1.size());
        replacedert.replacedertEquals("ch1", reqCh1.get(0).getName());
        replacedert.replacedertEquals("ch2", reqCh1.get(1).getName());
        List<Channel> optCh1 = selector.getOptionalChannels(event1);
        replacedert.replacedertTrue(optCh1.size() == 1);
        // ch2 should not be there -- since it is a required channel
        replacedert.replacedertEquals("ch3", optCh1.get(0).getName());
        Event event2 = new MockEvent();
        Map<String, String> header2 = new HashMap<String, String>();
        // should match ch2 ch3
        header2.put("myheader", "bar");
        event2.setHeaders(header2);
        List<Channel> reqCh2 = selector.getRequiredChannels(event2);
        replacedert.replacedertEquals(2, reqCh2.size());
        replacedert.replacedertEquals("ch2", reqCh2.get(0).getName());
        replacedert.replacedertEquals("ch3", reqCh2.get(1).getName());
        List<Channel> optCh2 = selector.getOptionalChannels(event2);
        replacedert.replacedertTrue(optCh2.isEmpty());
        Event event3 = new MockEvent();
        Map<String, String> header3 = new HashMap<String, String>();
        // should match ch1 ch2 ch3
        header3.put("myheader", "xyz");
        event3.setHeaders(header3);
        List<Channel> reqCh3 = selector.getRequiredChannels(event3);
        replacedert.replacedertEquals(3, reqCh3.size());
        replacedert.replacedertEquals("ch1", reqCh3.get(0).getName());
        replacedert.replacedertEquals("ch2", reqCh3.get(1).getName());
        replacedert.replacedertEquals("ch3", reqCh3.get(2).getName());
        List<Channel> optCh3 = selector.getOptionalChannels(event3);
        // All of the optional channels should go away.
        replacedert.replacedertTrue(optCh3.isEmpty());
        Event event4 = new MockEvent();
        Map<String, String> header4 = new HashMap<String, String>();
        header4.put("myheader", "zebra");
        event4.setHeaders(header4);
        List<Channel> reqCh4 = selector.getRequiredChannels(event4);
        replacedert.replacedertEquals(1, reqCh4.size());
        replacedert.replacedertEquals("ch2", reqCh4.get(0).getName());
        List<Channel> optCh4 = selector.getOptionalChannels(event4);
        replacedert.replacedertEquals(2, optCh4.size());
        replacedert.replacedertEquals("ch1", optCh4.get(0).getName());
        replacedert.replacedertEquals("ch3", optCh4.get(1).getName());
    }

    @Test
    public void testNoMandatory() {
        config.put("default", "ch3");
        config.put("optional.foo", "ch1 ch2");
        config.put("optional.zebra", "ch2 ch3");
        selector = ChannelSelectorFactory.create(channels, config);
        replacedert.replacedertTrue(selector instanceof MultiplexingChannelSelector);
        Event event1 = new MockEvent();
        Map<String, String> header1 = new HashMap<String, String>();
        // should match ch1 ch2
        header1.put("myheader", "foo");
        event1.setHeaders(header1);
        List<Channel> reqCh1 = selector.getRequiredChannels(event1);
        replacedert.replacedertEquals(1, reqCh1.size());
        replacedert.replacedertEquals("ch3", reqCh1.get(0).getName());
        List<Channel> optCh1 = selector.getOptionalChannels(event1);
        replacedert.replacedertEquals(2, optCh1.size());
        // ch2 should not be there -- since it is a required channel
        replacedert.replacedertEquals("ch1", optCh1.get(0).getName());
        replacedert.replacedertEquals("ch2", optCh1.get(1).getName());
        Event event4 = new MockEvent();
        Map<String, String> header4 = new HashMap<String, String>();
        header4.put("myheader", "zebra");
        event4.setHeaders(header4);
        List<Channel> reqCh4 = selector.getRequiredChannels(event4);
        replacedert.replacedertEquals(1, reqCh4.size());
        replacedert.replacedertTrue(reqCh4.get(0).getName().equals("ch3"));
        List<Channel> optCh4 = selector.getOptionalChannels(event4);
        // ch3 was returned as a required channel, because it is default.
        // So it is not returned in optional
        replacedert.replacedertEquals(1, optCh4.size());
        replacedert.replacedertEquals("ch2", optCh4.get(0).getName());
    }

    @Test
    public void testOnlyOptional() {
        config.put("optional.foo", "ch1 ch2");
        config.put("optional.zebra", "ch2 ch3");
        selector = ChannelSelectorFactory.create(channels, config);
        replacedert.replacedertTrue(selector instanceof MultiplexingChannelSelector);
        Event event1 = new MockEvent();
        Map<String, String> header1 = new HashMap<String, String>();
        // should match ch1 ch2
        header1.put("myheader", "foo");
        event1.setHeaders(header1);
        List<Channel> reqCh1 = selector.getRequiredChannels(event1);
        replacedert.replacedertTrue(reqCh1.isEmpty());
        List<Channel> optCh1 = selector.getOptionalChannels(event1);
        replacedert.replacedertEquals(2, optCh1.size());
        // ch2 should not be there -- since it is a required channel
        Event event4 = new MockEvent();
        Map<String, String> header4 = new HashMap<String, String>();
        header4.put("myheader", "zebra");
        event4.setHeaders(header4);
        List<Channel> reqCh4 = selector.getRequiredChannels(event4);
        replacedert.replacedertTrue(reqCh4.isEmpty());
        List<Channel> optCh4 = selector.getOptionalChannels(event4);
        replacedert.replacedertEquals(2, optCh4.size());
        replacedert.replacedertEquals("ch2", optCh4.get(0).getName());
        replacedert.replacedertEquals("ch3", optCh4.get(1).getName());
    }
}

18 View Complete Implementation : TestDefaultLogicalNodeManager.java
Copyright Apache License 2.0
Author : cloudera
@Test
public void testLifecycleWithNodes() throws LifecycleException, InterruptedException {
    nodeManager.start();
    replacedert.replacedertTrue("Node manager didn't reach START or ERROR", LifecycleController.waitForOneOf(nodeManager, LifecycleState.START_OR_ERROR, 5000));
    for (int i = 0; i < 3; i++) {
        SequenceGeneratorSource source = new SequenceGeneratorSource();
        List<Channel> channels = new ArrayList<Channel>();
        channels.add(new MemoryChannel());
        ChannelSelector rcs = new ReplicatingChannelSelector();
        rcs.setChannels(channels);
        source.setChannelProcessor(new ChannelProcessor(rcs));
        PollableSourceRunner sourceRunner = new PollableSourceRunner();
        sourceRunner.setSource(source);
        nodeManager.add(sourceRunner);
    }
    Thread.sleep(5000);
    nodeManager.stop();
    replacedert.replacedertTrue("Node manager didn't reach STOP or ERROR", LifecycleController.waitForOneOf(nodeManager, LifecycleState.STOP_OR_ERROR, 5000));
}

18 View Complete Implementation : TestHTTPSource.java
Copyright Apache License 2.0
Author : apache
private static void configureSourceAndChannel(HTTPSource source, Channel channel, Context context) {
    Context channelContext = new Context();
    channelContext.put("capacity", "100");
    Configurables.configure(channel, channelContext);
    Configurables.configure(source, context);
    ChannelSelector rcs1 = new ReplicatingChannelSelector();
    rcs1.setChannels(Collections.singletonList(channel));
    source.setChannelProcessor(new ChannelProcessor(rcs1));
}

18 View Complete Implementation : TestThriftSource.java
Copyright Apache License 2.0
Author : apache
private void configureSource() {
    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
}

18 View Complete Implementation : TestExecSource.java
Copyright Apache License 2.0
Author : javachen
public clreplaced TestExecSource {

    private AbstractSource source;

    private Channel channel = new MemoryChannel();

    private Context context = new Context();

    private ChannelSelector rcs = new ReplicatingChannelSelector();

    @Before
    public void setUp() {
        context.put("keep-alive", "1");
        context.put("capacity", "1000");
        context.put("transactionCapacity", "1000");
        Configurables.configure(channel, context);
        rcs.setChannels(Lists.newArrayList(channel));
        source = new ExecSource();
        source.setChannelProcessor(new ChannelProcessor(rcs));
    }

    @After
    public void tearDown() {
        source.stop();
        // Remove the MBean registered for Monitoring
        ObjectName objName = null;
        try {
            objName = new ObjectName("org.apache.flume.source" + ":type=" + source.getName());
            ManagementFactory.getPlatformMBeanServer().unregisterMBean(objName);
        } catch (Exception ex) {
            System.out.println("Failed to unregister the monitored counter: " + objName + ex.getMessage());
        }
    }

    @Test
    public void testProcess() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        context.put("command", "cat /etc/preplacedwd");
        context.put("keep-alive", "1");
        context.put("capacity", "1000");
        context.put("transactionCapacity", "1000");
        Configurables.configure(source, context);
        source.start();
        Transaction transaction = channel.getTransaction();
        transaction.begin();
        Event event;
        FileOutputStream outputStream = new FileOutputStream("/tmp/flume-execsource." + Thread.currentThread().getId());
        while ((event = channel.take()) != null) {
            outputStream.write(event.getBody());
            outputStream.write('\n');
        }
        outputStream.close();
        transaction.commit();
        transaction.close();
        File file1 = new File("/tmp/flume-execsource." + Thread.currentThread().getId());
        File file2 = new File("/etc/preplacedwd");
        replacedert.replacedertEquals(FileUtils.checksumCRC32(file1), FileUtils.checksumCRC32(file2));
        FileUtils.forceDelete(file1);
    }

    @Test
    public void testShellCommandSimple() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        runTestShellCmdHelper("/bin/bash -c", "seq 5", new String[] { "1", "2", "3", "4", "5" });
    }

    @Test
    public void testShellCommandBackTicks() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        // command with backticks
        runTestShellCmdHelper("/bin/bash -c", "echo `seq 5`", new String[] { "1 2 3 4 5" });
        runTestShellCmdHelper("/bin/bash -c", "echo $(seq 5)", new String[] { "1 2 3 4 5" });
    }

    @Test
    public void testShellCommandComplex() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        // command with wildcards & pipes
        String[] expected = { "1234", "abcd", "ijk", "xyz", "zzz" };
        // pipes
        runTestShellCmdHelper("/bin/bash -c", "echo zzz 1234 xyz abcd ijk | xargs -n1 echo | sort -f", expected);
    }

    @Test
    public void testShellCommandScript() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        // mini script
        runTestShellCmdHelper("/bin/bash -c", "for i in {1..5}; do echo $i;done", new String[] { "1", "2", "3", "4", "5" });
        // shell arithmetic
        runTestShellCmdHelper("/bin/bash -c", "if ((2+2>3)); then  echo good; else echo not good; fi", new String[] { "good" });
    }

    @Test
    public void testShellCommandEmbeddingAndEscaping() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        System.out.println("######### PWD = " + new java.io.File(".").getCanonicalPath());
        // mini script
        BufferedReader reader = new BufferedReader(new FileReader("src/test/resources/test_command.txt"));
        try {
            String command1 = reader.readLine();
            replacedert.replacedertNotNull(command1);
            String[] output1 = new String[] { "'1'", "\"2\"", "\\3", "\\4" };
            runTestShellCmdHelper("/bin/bash -c", command1, output1);
            String command2 = reader.readLine();
            replacedert.replacedertNotNull(command2);
            String[] output2 = new String[] { "1", "2", "3", "4", "5" };
            runTestShellCmdHelper("/bin/bash -c", command2, output2);
            String command3 = reader.readLine();
            replacedert.replacedertNotNull(command3);
            String[] output3 = new String[] { "2", "3", "4", "5", "6" };
            runTestShellCmdHelper("/bin/bash -c", command3, output3);
        } finally {
            reader.close();
        }
    }

    @Test
    public void testMonitoredCounterGroup() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        // mini script
        runTestShellCmdHelper("/bin/bash -c", "for i in {1..5}; do echo $i;done", new String[] { "1", "2", "3", "4", "5" });
        ObjectName objName = null;
        try {
            objName = new ObjectName("org.apache.flume.source" + ":type=" + source.getName());
            MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
            String[] strAtts = { "Type", "EventReceivedCount", "EventAcceptedCount" };
            AttributeList attrList = mbeanServer.getAttributes(objName, strAtts);
            replacedert.replacedertNotNull(attrList.get(0));
            replacedert.replacedertEquals("Expected Value: Type", "Type", ((Attribute) attrList.get(0)).getName());
            replacedert.replacedertEquals("Expected Value: SOURCE", "SOURCE", ((Attribute) attrList.get(0)).getValue());
            replacedert.replacedertNotNull(attrList.get(1));
            replacedert.replacedertEquals("Expected Value: EventReceivedCount", "EventReceivedCount", ((Attribute) attrList.get(1)).getName());
            replacedert.replacedertEquals("Expected Value: 5", "5", ((Attribute) attrList.get(1)).getValue().toString());
            replacedert.replacedertNotNull(attrList.get(2));
            replacedert.replacedertEquals("Expected Value: EventAcceptedCount", "EventAcceptedCount", ((Attribute) attrList.get(2)).getName());
            replacedert.replacedertEquals("Expected Value: 5", "5", ((Attribute) attrList.get(2)).getValue().toString());
        } catch (Exception ex) {
            System.out.println("Unable to retreive the monitored counter: " + objName + ex.getMessage());
        }
    }

    @Test
    public void testBatchTimeout() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        String filePath = "/tmp/flume-execsource." + Thread.currentThread().getId();
        String eventBody = "TestMessage";
        FileOutputStream outputStream = new FileOutputStream(filePath);
        context.put(ExecSourceConfigurationConstants.CONFIG_BATCH_SIZE, "50000");
        context.put(ExecSourceConfigurationConstants.CONFIG_BATCH_TIME_OUT, "750");
        context.put("shell", "/bin/bash -c");
        context.put("command", "tail -f " + filePath);
        Configurables.configure(source, context);
        source.start();
        Transaction transaction = channel.getTransaction();
        transaction.begin();
        for (int lineNumber = 0; lineNumber < 3; lineNumber++) {
            outputStream.write((eventBody).getBytes());
            outputStream.write(String.valueOf(lineNumber).getBytes());
            outputStream.write('\n');
            outputStream.flush();
        }
        outputStream.close();
        Thread.sleep(1500);
        for (int i = 0; i < 3; i++) {
            Event event = channel.take();
            replacedertNotNull(event);
            replacedertNotNull(event.getBody());
            replacedertEquals(eventBody + String.valueOf(i), new String(event.getBody()));
        }
        transaction.commit();
        transaction.close();
        source.stop();
        File file = new File(filePath);
        FileUtils.forceDelete(file);
    }

    private void runTestShellCmdHelper(String shell, String command, String[] expectedOutput) throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        context.put("shell", shell);
        context.put("command", command);
        Configurables.configure(source, context);
        source.start();
        File outputFile = File.createTempFile("flumeExecSourceTest_", "");
        FileOutputStream outputStream = new FileOutputStream(outputFile);
        Transaction transaction = channel.getTransaction();
        transaction.begin();
        try {
            Event event;
            while ((event = channel.take()) != null) {
                outputStream.write(event.getBody());
                outputStream.write('\n');
            }
            outputStream.close();
            transaction.commit();
            List<String> output = Files.readLines(outputFile, Charset.defaultCharset());
            System.out.println("command : " + command);
            System.out.println("output : ");
            for (String line : output) System.out.println();
            replacedert.replacedertArrayEquals(expectedOutput, output.toArray(new String[] {}));
        } finally {
            FileUtils.forceDelete(outputFile);
            transaction.close();
            source.stop();
        }
    }

    @Test
    public void testRestart() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
        context.put(ExecSourceConfigurationConstants.CONFIG_RESTART_THROTTLE, "10");
        context.put(ExecSourceConfigurationConstants.CONFIG_RESTART, "true");
        context.put("command", "echo flume");
        Configurables.configure(source, context);
        source.start();
        Transaction transaction = channel.getTransaction();
        transaction.begin();
        long start = System.currentTimeMillis();
        for (int i = 0; i < 5; i++) {
            Event event = channel.take();
            replacedertNotNull(event);
            replacedertNotNull(event.getBody());
            replacedertEquals("flume", new String(event.getBody(), Charsets.UTF_8));
        }
        // ensure restartThrottle was turned down as expected
        replacedertTrue(System.currentTimeMillis() - start < 10000L);
        transaction.commit();
        transaction.close();
        source.stop();
    }

    /**
     * Tests to make sure that the shutdown mechanism works. There are races
     * in this test if the system has another sleep command running with the
     * same sleep interval but we pick rarely used sleep times and make an
     * effort to detect if our sleep time is already in use. Note the
     * ps -ef command should work on both macs and linux.
     */
    @Test
    public void testShutdown() throws Exception {
        // pick a rare sleep time
        int seconds = 272;
        // now find one that is not in use
        boolean searchForCommand = true;
        while (searchForCommand) {
            searchForCommand = false;
            String command = "sleep " + seconds;
            Pattern pattern = Pattern.compile("\b" + command + "\b");
            for (String line : exec("ps -ef")) {
                if (pattern.matcher(line).find()) {
                    seconds++;
                    searchForCommand = true;
                    break;
                }
            }
        }
        // yes in the mean time someone could use our sleep time
        // but this should be a fairly rare scenerio
        String command = "sleep " + seconds;
        Pattern pattern = Pattern.compile("\b" + command + "\b");
        context.put(ExecSourceConfigurationConstants.CONFIG_RESTART, "false");
        context.put("command", command);
        Configurables.configure(source, context);
        source.start();
        Thread.sleep(1000L);
        source.stop();
        Thread.sleep(1000L);
        for (String line : exec("ps -ef")) {
            if (pattern.matcher(line).find()) {
                replacedert.fail("Found [" + line + "]");
            }
        }
    }

    private static List<String> exec(String command) throws Exception {
        String[] commandArgs = command.split("\\s+");
        Process process = new ProcessBuilder(commandArgs).start();
        BufferedReader reader = null;
        try {
            reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
            List<String> result = Lists.newArrayList();
            String line;
            while ((line = reader.readLine()) != null) {
                result.add(line);
            }
            return result;
        } finally {
            process.destroy();
            if (reader != null) {
                reader.close();
            }
            int exit = process.waitFor();
            if (exit != 0) {
                throw new IllegalStateException("Command [" + command + "] exited with " + exit);
            }
        }
    }
}

18 View Complete Implementation : TestTaildirSource.java
Copyright Apache License 2.0
Author : apache
@Before
public void setUp() {
    source = new TaildirSource();
    channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
    tmpDir = Files.createTempDir();
    posFilePath = tmpDir.getAbsolutePath() + "/taildir_position_test.json";
}

18 View Complete Implementation : TestReplicatingChannelSelector.java
Copyright Apache License 2.0
Author : apache
public clreplaced TestReplicatingChannelSelector {

    private List<Channel> channels = new ArrayList<Channel>();

    private ChannelSelector selector;

    @Before
    public void setUp() throws Exception {
        channels.clear();
        channels.add(MockChannel.createMockChannel("ch1"));
        channels.add(MockChannel.createMockChannel("ch2"));
        channels.add(MockChannel.createMockChannel("ch3"));
        channels.add(MockChannel.createMockChannel("ch4"));
        selector = ChannelSelectorFactory.create(channels, new HashMap<String, String>());
    }

    @Test
    public void testReplicatingSelector() throws Exception {
        selector.configure(new Context());
        List<Channel> channels = selector.getRequiredChannels(new MockEvent());
        replacedert.replacedertNotNull(channels);
        replacedert.replacedertEquals(4, channels.size());
        replacedert.replacedertEquals("ch1", channels.get(0).getName());
        replacedert.replacedertEquals("ch2", channels.get(1).getName());
        replacedert.replacedertEquals("ch3", channels.get(2).getName());
        replacedert.replacedertEquals("ch4", channels.get(3).getName());
        List<Channel> optCh = selector.getOptionalChannels(new MockEvent());
        replacedert.replacedertEquals(0, optCh.size());
    }

    @Test
    public void testOptionalChannels() throws Exception {
        Context context = new Context();
        context.put(ReplicatingChannelSelector.CONFIG_OPTIONAL, "ch1");
        Configurables.configure(selector, context);
        List<Channel> channels = selector.getRequiredChannels(new MockEvent());
        replacedert.replacedertNotNull(channels);
        replacedert.replacedertEquals(3, channels.size());
        replacedert.replacedertEquals("ch2", channels.get(0).getName());
        replacedert.replacedertEquals("ch3", channels.get(1).getName());
        replacedert.replacedertEquals("ch4", channels.get(2).getName());
        List<Channel> optCh = selector.getOptionalChannels(new MockEvent());
        replacedert.replacedertEquals(1, optCh.size());
        replacedert.replacedertEquals("ch1", optCh.get(0).getName());
    }

    @Test
    public void testMultipleOptionalChannels() throws Exception {
        Context context = new Context();
        context.put(ReplicatingChannelSelector.CONFIG_OPTIONAL, "ch1 ch4");
        Configurables.configure(selector, context);
        List<Channel> channels = selector.getRequiredChannels(new MockEvent());
        replacedert.replacedertNotNull(channels);
        replacedert.replacedertEquals(2, channels.size());
        replacedert.replacedertEquals("ch2", channels.get(0).getName());
        replacedert.replacedertEquals("ch3", channels.get(1).getName());
        List<Channel> optCh = selector.getOptionalChannels(new MockEvent());
        replacedert.replacedertEquals(2, optCh.size());
        replacedert.replacedertEquals("ch1", optCh.get(0).getName());
        replacedert.replacedertEquals("ch4", optCh.get(1).getName());
    }

    @Test
    public void testMultipleOptionalChannelsSameChannelTwice() throws Exception {
        Context context = new Context();
        context.put(ReplicatingChannelSelector.CONFIG_OPTIONAL, "ch1 ch4 ch1");
        Configurables.configure(selector, context);
        List<Channel> channels = selector.getRequiredChannels(new MockEvent());
        replacedert.replacedertNotNull(channels);
        replacedert.replacedertEquals(2, channels.size());
        replacedert.replacedertEquals("ch2", channels.get(0).getName());
        replacedert.replacedertEquals("ch3", channels.get(1).getName());
        List<Channel> optCh = selector.getOptionalChannels(new MockEvent());
        replacedert.replacedertEquals(2, optCh.size());
        replacedert.replacedertEquals("ch1", optCh.get(0).getName());
        replacedert.replacedertEquals("ch4", optCh.get(1).getName());
    }
}

18 View Complete Implementation : ChannelSelectorFactory.java
Copyright Apache License 2.0
Author : cloudera
private static ChannelSelector getSelectorForType(String type) {
    if (type == null || type.trim().length() == 0) {
        return new ReplicatingChannelSelector();
    }
    String selectorClreplacedName = type;
    ChannelSelectorType selectorType = ChannelSelectorType.OTHER;
    try {
        selectorType = ChannelSelectorType.valueOf(type.toUpperCase());
    } catch (IllegalArgumentException ex) {
        LOGGER.debug("Selector type {} is a custom type", type);
    }
    if (!selectorType.equals(ChannelSelectorType.OTHER)) {
        selectorClreplacedName = selectorType.getChannelSelectorClreplacedName();
    }
    ChannelSelector selector = null;
    try {
        @SuppressWarnings("unchecked")
        Clreplaced<? extends ChannelSelector> selectorClreplaced = (Clreplaced<? extends ChannelSelector>) Clreplaced.forName(selectorClreplacedName);
        selector = selectorClreplaced.newInstance();
    } catch (Exception ex) {
        throw new FlumeException("Unable to load selector type: " + type + ", clreplaced: " + selectorClreplacedName, ex);
    }
    return selector;
}

17 View Complete Implementation : ChannelProcessor.java
Copyright Apache License 2.0
Author : javachen
/**
 * A channel processor exposes operations to put {@link Event}s into
 * {@link Channel}s. These operations will propagate a {@link ChannelException}
 * if any errors occur while attempting to write to {@code required} channels.
 *
 * Each channel processor instance is configured with a {@link ChannelSelector}
 * instance that specifies which channels are
 * {@linkplain ChannelSelector#getRequiredChannels(Event) required} and which
 * channels are
 * {@linkplain ChannelSelector#getOptionalChannels(Event) optional}.
 */
public clreplaced ChannelProcessor implements Configurable {

    private static final Logger LOG = LoggerFactory.getLogger(ChannelProcessor.clreplaced);

    private final ChannelSelector selector;

    private final InterceptorChain interceptorChain;

    public ChannelProcessor(ChannelSelector selector) {
        this.selector = selector;
        this.interceptorChain = new InterceptorChain();
    }

    public void initialize() {
        interceptorChain.initialize();
    }

    public void close() {
        interceptorChain.close();
    }

    /**
     * The Context of the replacedociated Source is preplaceded.
     * @param context
     */
    @Override
    public void configure(Context context) {
        configureInterceptors(context);
    }

    // WARNING: throws FlumeException (is that ok?)
    private void configureInterceptors(Context context) {
        List<Interceptor> interceptors = Lists.newLinkedList();
        String interceptorListStr = context.getString("interceptors", "");
        if (interceptorListStr.isEmpty()) {
            return;
        }
        String[] interceptorNames = interceptorListStr.split("\\s+");
        Context interceptorContexts = new Context(context.getSubProperties("interceptors."));
        // run through and instantiate all the interceptors specified in the Context
        InterceptorBuilderFactory factory = new InterceptorBuilderFactory();
        for (String interceptorName : interceptorNames) {
            Context interceptorContext = new Context(interceptorContexts.getSubProperties(interceptorName + "."));
            String type = interceptorContext.getString("type");
            if (type == null) {
                LOG.error("Type not specified for interceptor " + interceptorName);
                throw new FlumeException("Interceptor.Type not specified for " + interceptorName);
            }
            try {
                Interceptor.Builder builder = factory.newInstance(type);
                builder.configure(interceptorContext);
                interceptors.add(builder.build());
            } catch (ClreplacedNotFoundException e) {
                LOG.error("Builder clreplaced not found. Exception follows.", e);
                throw new FlumeException("Interceptor.Builder not found.", e);
            } catch (InstantiationException e) {
                LOG.error("Could not instantiate Builder. Exception follows.", e);
                throw new FlumeException("Interceptor.Builder not constructable.", e);
            } catch (IllegalAccessException e) {
                LOG.error("Unable to access Builder. Exception follows.", e);
                throw new FlumeException("Unable to access Interceptor.Builder.", e);
            }
        }
        interceptorChain.setInterceptors(interceptors);
    }

    public ChannelSelector getSelector() {
        return selector;
    }

    /**
     * Attempts to {@linkplain Channel#put(Event) put} the given events into each
     * configured channel. If any {@code required} channel throws a
     * {@link ChannelException}, that exception will be propagated.
     *
     * <p>Note that if multiple channels are configured, some {@link Transaction}s
     * may have already been committed while others may be rolled back in the
     * case of an exception.
     *
     * @param events A list of events to put into the configured channels.
     * @throws ChannelException when a write to a required channel fails.
     */
    public void processEventBatch(List<Event> events) {
        Preconditions.checkNotNull(events, "Event list must not be null");
        events = interceptorChain.intercept(events);
        Map<Channel, List<Event>> reqChannelQueue = new LinkedHashMap<Channel, List<Event>>();
        Map<Channel, List<Event>> optChannelQueue = new LinkedHashMap<Channel, List<Event>>();
        for (Event event : events) {
            List<Channel> reqChannels = selector.getRequiredChannels(event);
            for (Channel ch : reqChannels) {
                List<Event> eventQueue = reqChannelQueue.get(ch);
                if (eventQueue == null) {
                    eventQueue = new ArrayList<Event>();
                    reqChannelQueue.put(ch, eventQueue);
                }
                eventQueue.add(event);
            }
            List<Channel> optChannels = selector.getOptionalChannels(event);
            for (Channel ch : optChannels) {
                List<Event> eventQueue = optChannelQueue.get(ch);
                if (eventQueue == null) {
                    eventQueue = new ArrayList<Event>();
                    optChannelQueue.put(ch, eventQueue);
                }
                eventQueue.add(event);
            }
        }
        // Process required channels
        for (Channel reqChannel : reqChannelQueue.keySet()) {
            Transaction tx = reqChannel.getTransaction();
            Preconditions.checkNotNull(tx, "Transaction object must not be null");
            try {
                tx.begin();
                List<Event> batch = reqChannelQueue.get(reqChannel);
                for (Event event : batch) {
                    reqChannel.put(event);
                }
                tx.commit();
            } catch (Throwable t) {
                tx.rollback();
                if (t instanceof Error) {
                    LOG.error("Error while writing to required channel: " + reqChannel, t);
                    throw (Error) t;
                } else {
                    throw new ChannelException("Unable to put batch on required " + "channel: " + reqChannel, t);
                }
            } finally {
                if (tx != null) {
                    tx.close();
                }
            }
        }
        // Process optional channels
        for (Channel optChannel : optChannelQueue.keySet()) {
            Transaction tx = optChannel.getTransaction();
            Preconditions.checkNotNull(tx, "Transaction object must not be null");
            try {
                tx.begin();
                List<Event> batch = optChannelQueue.get(optChannel);
                for (Event event : batch) {
                    optChannel.put(event);
                }
                tx.commit();
            } catch (Throwable t) {
                tx.rollback();
                LOG.error("Unable to put batch on optional channel: " + optChannel, t);
                if (t instanceof Error) {
                    throw (Error) t;
                }
            } finally {
                if (tx != null) {
                    tx.close();
                }
            }
        }
    }

    /**
     * Attempts to {@linkplain Channel#put(Event) put} the given event into each
     * configured channel. If any {@code required} channel throws a
     * {@link ChannelException}, that exception will be propagated.
     *
     * <p>Note that if multiple channels are configured, some {@link Transaction}s
     * may have already been committed while others may be rolled back in the
     * case of an exception.
     *
     * @param event The event to put into the configured channels.
     * @throws ChannelException when a write to a required channel fails.
     */
    public void processEvent(Event event) {
        event = interceptorChain.intercept(event);
        if (event == null) {
            return;
        }
        // Process required channels
        List<Channel> requiredChannels = selector.getRequiredChannels(event);
        for (Channel reqChannel : requiredChannels) {
            Transaction tx = reqChannel.getTransaction();
            Preconditions.checkNotNull(tx, "Transaction object must not be null");
            try {
                tx.begin();
                reqChannel.put(event);
                tx.commit();
            } catch (Throwable t) {
                tx.rollback();
                if (t instanceof Error) {
                    LOG.error("Error while writing to required channel: " + reqChannel, t);
                    throw (Error) t;
                } else {
                    throw new ChannelException("Unable to put event on required " + "channel: " + reqChannel, t);
                }
            } finally {
                if (tx != null) {
                    tx.close();
                }
            }
        }
        // Process optional channels
        List<Channel> optionalChannels = selector.getOptionalChannels(event);
        for (Channel optChannel : optionalChannels) {
            Transaction tx = null;
            try {
                tx = optChannel.getTransaction();
                tx.begin();
                optChannel.put(event);
                tx.commit();
            } catch (Throwable t) {
                tx.rollback();
                LOG.error("Unable to put event on optional channel: " + optChannel, t);
                if (t instanceof Error) {
                    throw (Error) t;
                }
            } finally {
                if (tx != null) {
                    tx.close();
                }
            }
        }
    }
}

17 View Complete Implementation : TestLoadBalancingLog4jAppender.java
Copyright Apache License 2.0
Author : kite-sdk
public clreplaced TestLoadBalancingLog4replacedpender {

    private final List<CountingAvroSource> sources = Lists.newArrayList();

    private Channel ch;

    private ChannelSelector rcs;

    private Logger fixture;

    @Before
    public void initiate() throws InterruptedException {
        ch = new MemoryChannel();
        Configurables.configure(ch, new Context());
        List<Channel> channels = new ArrayList<Channel>();
        channels.add(ch);
        rcs = new ReplicatingChannelSelector();
        rcs.setChannels(channels);
    }

    @After
    public void cleanUp() {
        for (Source source : sources) {
            source.stop();
        }
    }

    @Test
    public void testLog4replacedpenderRoundRobin() throws IOException {
        int numberOfMsgs = 1000;
        int expectedPerSource = 500;
        File TESTFILE = new File(TestLoadBalancingLog4replacedpender.clreplaced.getClreplacedLoader().getResource("flume-loadbalancinglog4jtest.properties").getFile());
        startSources(TESTFILE, new int[] { 25430, 25431 });
        sendAndreplacedertMessages(numberOfMsgs);
        for (CountingAvroSource source : sources) {
            replacedert.replacedertEquals(expectedPerSource, source.appendCount.get());
        }
    }

    @Test
    public void testLog4replacedpenderRandom() throws IOException {
        int numberOfMsgs = 1000;
        File TESTFILE = new File(TestLoadBalancingLog4replacedpender.clreplaced.getClreplacedLoader().getResource("flume-loadbalancing-rnd-log4jtest.properties").getFile());
        startSources(TESTFILE, new int[] { 25430, 25431, 25432, 25433, 25434, 25435, 25436, 25437, 25438, 25439 });
        sendAndreplacedertMessages(numberOfMsgs);
        int total = 0;
        Set<Integer> counts = new HashSet<Integer>();
        for (CountingAvroSource source : sources) {
            total += source.appendCount.intValue();
            counts.add(source.appendCount.intValue());
        }
        // We are not testing distribution this is tested in the client
        replacedert.replacedertTrue("Very unusual distribution " + counts.size(), counts.size() > 2);
        replacedert.replacedertTrue("Missing events", total == numberOfMsgs);
    }

    @Test
    public void testRandomBackoff() throws Exception {
        File TESTFILE = new File(TestLoadBalancingLog4replacedpender.clreplaced.getClreplacedLoader().getResource("flume-loadbalancing-backoff-log4jtest.properties").getFile());
        startSources(TESTFILE, new int[] { 25430, 25431, 25432 });
        sources.get(0).setFail();
        sources.get(2).setFail();
        sendAndreplacedertMessages(50);
        replacedert.replacedertEquals(50, sources.get(1).appendCount.intValue());
        replacedert.replacedertEquals(0, sources.get(0).appendCount.intValue());
        replacedert.replacedertEquals(0, sources.get(2).appendCount.intValue());
        sources.get(0).setOk();
        // s0 should still be backed off
        sources.get(1).setFail();
        try {
            send(1);
            // nothing should be able to process right now
            replacedert.fail("Expected EventDeliveryException");
        } catch (FlumeException e) {
            replacedert.replacedertTrue(e.getCause() instanceof EventDeliveryException);
        }
        // wait for s0 to no longer be backed off
        Thread.sleep(2500);
        sendAndreplacedertMessages(50);
        replacedert.replacedertEquals(50, sources.get(0).appendCount.intValue());
        replacedert.replacedertEquals(50, sources.get(1).appendCount.intValue());
        replacedert.replacedertEquals(0, sources.get(2).appendCount.intValue());
    }

    private void send(int numberOfMsgs) throws EventDeliveryException {
        for (int count = 0; count < numberOfMsgs; count++) {
            int level = count % 5;
            String msg = "This is log message number" + String.valueOf(count);
            fixture.log(Level.toLevel(level), msg);
        }
    }

    private void sendAndreplacedertMessages(int numberOfMsgs) throws IOException {
        for (int count = 0; count < numberOfMsgs; count++) {
            int level = count % 5;
            String msg = "This is log message number" + String.valueOf(count);
            fixture.log(Level.toLevel(level), msg);
            Transaction transaction = ch.getTransaction();
            transaction.begin();
            Event event = ch.take();
            replacedert.replacedertNotNull(event);
            replacedert.replacedertEquals(new String(event.getBody(), "UTF8"), msg);
            Map<String, String> hdrs = event.getHeaders();
            replacedert.replacedertNotNull(hdrs.get(Log4jAvroHeaders.TIMESTAMP.toString()));
            replacedert.replacedertEquals(Level.toLevel(level), Level.toLevel(hdrs.get(Log4jAvroHeaders.LOG_LEVEL.toString())));
            replacedert.replacedertEquals(fixture.getName(), hdrs.get(Log4jAvroHeaders.LOGGER_NAME.toString()));
            replacedert.replacedertEquals("UTF8", hdrs.get(Log4jAvroHeaders.MESSAGE_ENCODING.toString()));
            // To confirm on console we actually got the body
            System.out.println("Got body: " + new String(event.getBody(), "UTF8"));
            transaction.commit();
            transaction.close();
        }
    }

    private void startSources(File log4jProps, int... ports) throws IOException {
        for (int port : ports) {
            CountingAvroSource source = new CountingAvroSource(port);
            Context context = new Context();
            context.put("port", String.valueOf(port));
            context.put("bind", "0.0.0.0");
            Configurables.configure(source, context);
            sources.add(source);
            source.setChannelProcessor(new ChannelProcessor(rcs));
        }
        for (Source source : sources) {
            source.start();
        }
        // The properties file having Avro port info should be loaded only
        // after the test begins, else log4j tries to connect to the source
        // before the source has started up in the above function, since
        // log4j setup is completed before the @Before calls also.
        // This will cause the test to fail even before it starts!
        FileReader reader = new FileReader(log4jProps);
        Properties props = new Properties();
        props.load(reader);
        PropertyConfigurator.configure(props);
        fixture = LogManager.getLogger(TestLoadBalancingLog4replacedpender.clreplaced);
    }

    static clreplaced CountingAvroSource extends AvroSource {

        AtomicInteger appendCount = new AtomicInteger();

        volatile boolean isFail = false;

        private final int port2;

        public CountingAvroSource(int port) {
            port2 = port;
        }

        public void setOk() {
            this.isFail = false;
        }

        public void setFail() {
            this.isFail = true;
        }

        @Override
        public String getName() {
            return "testing..." + port2;
        }

        @Override
        public Status append(AvroFlumeEvent avroEvent) {
            if (isFail) {
                return Status.FAILED;
            }
            appendCount.incrementAndGet();
            return super.append(avroEvent);
        }

        @Override
        public Status appendBatch(List<AvroFlumeEvent> events) {
            if (isFail) {
                return Status.FAILED;
            }
            appendCount.addAndGet(events.size());
            return super.appendBatch(events);
        }
    }
}

17 View Complete Implementation : TestLoadBalancingLog4jAppender.java
Copyright Apache License 2.0
Author : apache
public clreplaced TestLoadBalancingLog4replacedpender {

    private final List<CountingAvroSource> sources = Lists.newArrayList();

    private Channel ch;

    private ChannelSelector rcs;

    private Logger fixture;

    private boolean slowDown = false;

    private static List<Integer> getFreePorts(int numberOfPorts) throws IOException {
        List<Integer> ports = new ArrayList<>(numberOfPorts);
        for (int index = 0; index < numberOfPorts; ++index) {
            try (ServerSocket socket = new ServerSocket(0)) {
                ports.add(socket.getLocalPort());
            }
        }
        return ports;
    }

    private static String toHostList(List<Integer> ports) {
        List<String> addresses = new ArrayList<String>(ports.size());
        for (Integer port : ports) {
            addresses.add("localhost:" + port);
        }
        String hostList = StringUtils.join(addresses, " ");
        return hostList;
    }

    @Before
    public void initiate() throws InterruptedException {
        ch = new MemoryChannel();
        configureChannel();
    }

    private void configureChannel() {
        Configurables.configure(ch, new Context());
        List<Channel> channels = new ArrayList<Channel>();
        channels.add(ch);
        rcs = new ReplicatingChannelSelector();
        rcs.setChannels(channels);
    }

    @After
    public void cleanUp() {
        for (Source source : sources) {
            source.stop();
        }
    }

    @Test
    public void testLog4replacedpenderRoundRobin() throws IOException {
        int numberOfMsgs = 1000;
        int expectedPerSource = 500;
        String propertiesFile = "flume-loadbalancinglog4jtest.properties";
        startSources(propertiesFile, false, getFreePorts(2));
        sendAndreplacedertMessages(numberOfMsgs);
        for (CountingAvroSource source : sources) {
            replacedert.replacedertEquals(expectedPerSource, source.appendCount.get());
        }
    }

    @Test
    public void testLog4replacedpenderRandom() throws IOException {
        int numberOfMsgs = 1000;
        String propertiesFile = "flume-loadbalancing-rnd-log4jtest.properties";
        startSources(propertiesFile, false, getFreePorts(10));
        sendAndreplacedertMessages(numberOfMsgs);
        int total = 0;
        Set<Integer> counts = new HashSet<Integer>();
        for (CountingAvroSource source : sources) {
            total += source.appendCount.intValue();
            counts.add(source.appendCount.intValue());
        }
        // We are not testing distribution this is tested in the client
        replacedert.replacedertTrue("Very unusual distribution " + counts.size(), counts.size() > 2);
        replacedert.replacedertTrue("Missing events", total == numberOfMsgs);
    }

    @Test
    public void testRandomBackoff() throws Exception {
        String propertiesFile = "flume-loadbalancing-backoff-log4jtest.properties";
        startSources(propertiesFile, false, getFreePorts(3));
        sources.get(0).setFail();
        sources.get(2).setFail();
        sendAndreplacedertMessages(50);
        replacedert.replacedertEquals(50, sources.get(1).appendCount.intValue());
        replacedert.replacedertEquals(0, sources.get(0).appendCount.intValue());
        replacedert.replacedertEquals(0, sources.get(2).appendCount.intValue());
        sources.get(0).setOk();
        // s0 should still be backed off
        sources.get(1).setFail();
        try {
            send(1);
            // nothing should be able to process right now
            replacedert.fail("Expected EventDeliveryException");
        } catch (FlumeException e) {
            replacedert.replacedertTrue(e.getCause() instanceof EventDeliveryException);
        }
        // wait for s0 to no longer be backed off
        Thread.sleep(2500);
        sendAndreplacedertMessages(50);
        replacedert.replacedertEquals(50, sources.get(0).appendCount.intValue());
        replacedert.replacedertEquals(50, sources.get(1).appendCount.intValue());
        replacedert.replacedertEquals(0, sources.get(2).appendCount.intValue());
    }

    @Test
    public void testRandomBackoffUnsafeMode() throws Exception {
        String propertiesFile = "flume-loadbalancing-backoff-log4jtest.properties";
        startSources(propertiesFile, true, getFreePorts(3));
        sources.get(0).setFail();
        sources.get(1).setFail();
        sources.get(2).setFail();
        sendAndreplacedertFail();
    }

    @Test(expected = EventDeliveryException.clreplaced)
    public void testTimeout() throws Throwable {
        String propertiesFile = "flume-loadbalancinglog4jtest.properties";
        ch = new TestLog4replacedpender.SlowMemoryChannel(2000);
        configureChannel();
        slowDown = true;
        startSources(propertiesFile, false, getFreePorts(3));
        int level = 20000;
        String msg = "This is log message number" + String.valueOf(level);
        try {
            fixture.log(Level.toLevel(level), msg);
        } catch (FlumeException ex) {
            throw ex.getCause();
        }
    }

    @Test(expected = EventDeliveryException.clreplaced)
    public void testRandomBackoffNotUnsafeMode() throws Throwable {
        String propertiesFile = "flume-loadbalancing-backoff-log4jtest.properties";
        startSources(propertiesFile, false, getFreePorts(3));
        sources.get(0).setFail();
        sources.get(1).setFail();
        sources.get(2).setFail();
        try {
            sendAndreplacedertFail();
        } catch (FlumeException ex) {
            throw ex.getCause();
        }
    }

    private void send(int numberOfMsgs) throws EventDeliveryException {
        for (int count = 0; count < numberOfMsgs; count++) {
            int level = count % 5;
            String msg = "This is log message number" + String.valueOf(count);
            fixture.log(Level.toLevel(level), msg);
        }
    }

    private void sendAndreplacedertFail() throws IOException {
        int level = 20000;
        String msg = "This is log message number" + String.valueOf(level);
        fixture.log(Level.toLevel(level), msg);
        Transaction transaction = ch.getTransaction();
        transaction.begin();
        Event event = ch.take();
        replacedert.replacedertNull(event);
        transaction.commit();
        transaction.close();
    }

    private void sendAndreplacedertMessages(int numberOfMsgs) throws IOException {
        for (int count = 0; count < numberOfMsgs; count++) {
            int level = count % 5;
            String msg = "This is log message number" + String.valueOf(count);
            fixture.log(Level.toLevel(level), msg);
            Transaction transaction = ch.getTransaction();
            transaction.begin();
            Event event = ch.take();
            replacedert.replacedertNotNull(event);
            replacedert.replacedertEquals(new String(event.getBody(), "UTF8"), msg);
            Map<String, String> hdrs = event.getHeaders();
            replacedert.replacedertNotNull(hdrs.get(Log4jAvroHeaders.TIMESTAMP.toString()));
            replacedert.replacedertNotNull(hdrs.get(Log4jAvroHeaders.ADDRESS.toString()));
            replacedert.replacedertEquals(Level.toLevel(level), Level.toLevel(hdrs.get(Log4jAvroHeaders.LOG_LEVEL.toString())));
            replacedert.replacedertEquals(fixture.getName(), hdrs.get(Log4jAvroHeaders.LOGGER_NAME.toString()));
            replacedert.replacedertEquals("UTF8", hdrs.get(Log4jAvroHeaders.MESSAGE_ENCODING.toString()));
            // To confirm on console we actually got the body
            System.out.println("Got body: " + new String(event.getBody(), "UTF8"));
            transaction.commit();
            transaction.close();
        }
    }

    private void startSources(String log4jProps, boolean unsafeMode, List<Integer> ports) throws IOException {
        for (int port : ports) {
            CountingAvroSource source = new CountingAvroSource(port);
            Context context = new Context();
            context.put("port", String.valueOf(port));
            context.put("bind", "0.0.0.0");
            Configurables.configure(source, context);
            sources.add(source);
            source.setChannelProcessor(new ChannelProcessor(rcs));
        }
        for (Source source : sources) {
            source.start();
        }
        // The properties file having Avro port info should be loaded only
        // after the test begins, else log4j tries to connect to the source
        // before the source has started up in the above function, since
        // log4j setup is completed before the @Before calls also.
        // This will cause the test to fail even before it starts!
        Reader reader = new InputStreamReader(getClreplaced().getResourcereplacedtream("/" + log4jProps));
        Properties props = new Properties();
        props.load(reader);
        props.setProperty("log4j.appender.out2.Hosts", toHostList(ports));
        props.setProperty("log4j.appender.out2.UnsafeMode", String.valueOf(unsafeMode));
        if (slowDown) {
            props.setProperty("log4j.appender.out2.Timeout", String.valueOf(1000));
        }
        PropertyConfigurator.configure(props);
        fixture = LogManager.getLogger(TestLoadBalancingLog4replacedpender.clreplaced);
    }

    static clreplaced CountingAvroSource extends AvroSource {

        AtomicInteger appendCount = new AtomicInteger();

        volatile boolean isFail = false;

        private final int port2;

        public CountingAvroSource(int port) {
            port2 = port;
        }

        public void setOk() {
            this.isFail = false;
        }

        public void setFail() {
            this.isFail = true;
        }

        @Override
        public String getName() {
            return "testing..." + port2;
        }

        @Override
        public Status append(AvroFlumeEvent avroEvent) {
            if (isFail) {
                return Status.FAILED;
            }
            appendCount.incrementAndGet();
            return super.append(avroEvent);
        }

        @Override
        public Status appendBatch(List<AvroFlumeEvent> events) {
            if (isFail) {
                return Status.FAILED;
            }
            appendCount.addAndGet(events.size());
            return super.appendBatch(events);
        }
    }
}

17 View Complete Implementation : TestLog4jAppender.java
Copyright Apache License 2.0
Author : apache
private void configureSource() {
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(Collections.singletonList(ch));
    source.setChannelProcessor(new ChannelProcessor(rcs));
    source.start();
}

17 View Complete Implementation : ChannelProcessor.java
Copyright Apache License 2.0
Author : apache
/**
 * A channel processor exposes operations to put {@link Event}s into
 * {@link Channel}s. These operations will propagate a {@link ChannelException}
 * if any errors occur while attempting to write to {@code required} channels.
 * <p>
 * Each channel processor instance is configured with a {@link ChannelSelector}
 * instance that specifies which channels are
 * {@linkplain ChannelSelector#getRequiredChannels(Event) required} and which
 * channels are
 * {@linkplain ChannelSelector#getOptionalChannels(Event) optional}.
 */
public clreplaced ChannelProcessor implements Configurable {

    private static final Logger LOG = LoggerFactory.getLogger(ChannelProcessor.clreplaced);

    private final ChannelSelector selector;

    private final InterceptorChain interceptorChain;

    public ChannelProcessor(ChannelSelector selector) {
        this.selector = selector;
        this.interceptorChain = new InterceptorChain();
    }

    public void initialize() {
        interceptorChain.initialize();
    }

    public void close() {
        interceptorChain.close();
    }

    /**
     * The Context of the replacedociated Source is preplaceded.
     *
     * @param context
     */
    @Override
    public void configure(Context context) {
        configureInterceptors(context);
    }

    // WARNING: throws FlumeException (is that ok?)
    private void configureInterceptors(Context context) {
        List<Interceptor> interceptors = Lists.newLinkedList();
        String interceptorListStr = context.getString("interceptors", "");
        if (interceptorListStr.isEmpty()) {
            return;
        }
        String[] interceptorNames = interceptorListStr.split("\\s+");
        Context interceptorContexts = new Context(context.getSubProperties("interceptors."));
        // run through and instantiate all the interceptors specified in the Context
        InterceptorBuilderFactory factory = new InterceptorBuilderFactory();
        for (String interceptorName : interceptorNames) {
            Context interceptorContext = new Context(interceptorContexts.getSubProperties(interceptorName + "."));
            String type = interceptorContext.getString("type");
            if (type == null) {
                LOG.error("Type not specified for interceptor " + interceptorName);
                throw new FlumeException("Interceptor.Type not specified for " + interceptorName);
            }
            try {
                Interceptor.Builder builder = factory.newInstance(type);
                builder.configure(interceptorContext);
                interceptors.add(builder.build());
            } catch (ClreplacedNotFoundException e) {
                LOG.error("Builder clreplaced not found. Exception follows.", e);
                throw new FlumeException("Interceptor.Builder not found.", e);
            } catch (InstantiationException e) {
                LOG.error("Could not instantiate Builder. Exception follows.", e);
                throw new FlumeException("Interceptor.Builder not constructable.", e);
            } catch (IllegalAccessException e) {
                LOG.error("Unable to access Builder. Exception follows.", e);
                throw new FlumeException("Unable to access Interceptor.Builder.", e);
            }
        }
        interceptorChain.setInterceptors(interceptors);
    }

    public ChannelSelector getSelector() {
        return selector;
    }

    /**
     * Attempts to {@linkplain Channel#put(Event) put} the given events into each
     * configured channel. If any {@code required} channel throws a
     * {@link ChannelException}, that exception will be propagated.
     * <p>
     * <p>Note that if multiple channels are configured, some {@link Transaction}s
     * may have already been committed while others may be rolled back in the
     * case of an exception.
     *
     * @param events A list of events to put into the configured channels.
     * @throws ChannelException when a write to a required channel fails.
     */
    public void processEventBatch(List<Event> events) {
        Preconditions.checkNotNull(events, "Event list must not be null");
        events = interceptorChain.intercept(events);
        Map<Channel, List<Event>> reqChannelQueue = new LinkedHashMap<Channel, List<Event>>();
        Map<Channel, List<Event>> optChannelQueue = new LinkedHashMap<Channel, List<Event>>();
        for (Event event : events) {
            List<Channel> reqChannels = selector.getRequiredChannels(event);
            for (Channel ch : reqChannels) {
                List<Event> eventQueue = reqChannelQueue.get(ch);
                if (eventQueue == null) {
                    eventQueue = new ArrayList<Event>();
                    reqChannelQueue.put(ch, eventQueue);
                }
                eventQueue.add(event);
            }
            List<Channel> optChannels = selector.getOptionalChannels(event);
            for (Channel ch : optChannels) {
                List<Event> eventQueue = optChannelQueue.get(ch);
                if (eventQueue == null) {
                    eventQueue = new ArrayList<Event>();
                    optChannelQueue.put(ch, eventQueue);
                }
                eventQueue.add(event);
            }
        }
        // Process required channels
        for (Channel reqChannel : reqChannelQueue.keySet()) {
            Transaction tx = reqChannel.getTransaction();
            Preconditions.checkNotNull(tx, "Transaction object must not be null");
            try {
                tx.begin();
                List<Event> batch = reqChannelQueue.get(reqChannel);
                for (Event event : batch) {
                    reqChannel.put(event);
                }
                tx.commit();
            } catch (Throwable t) {
                tx.rollback();
                if (t instanceof Error) {
                    LOG.error("Error while writing to required channel: " + reqChannel, t);
                    throw (Error) t;
                } else if (t instanceof ChannelException) {
                    throw (ChannelException) t;
                } else {
                    throw new ChannelException("Unable to put batch on required " + "channel: " + reqChannel, t);
                }
            } finally {
                if (tx != null) {
                    tx.close();
                }
            }
        }
        // Process optional channels
        for (Channel optChannel : optChannelQueue.keySet()) {
            Transaction tx = optChannel.getTransaction();
            Preconditions.checkNotNull(tx, "Transaction object must not be null");
            try {
                tx.begin();
                List<Event> batch = optChannelQueue.get(optChannel);
                for (Event event : batch) {
                    optChannel.put(event);
                }
                tx.commit();
            } catch (Throwable t) {
                tx.rollback();
                LOG.error("Unable to put batch on optional channel: " + optChannel, t);
                if (t instanceof Error) {
                    throw (Error) t;
                }
            } finally {
                if (tx != null) {
                    tx.close();
                }
            }
        }
    }

    /**
     * Attempts to {@linkplain Channel#put(Event) put} the given event into each
     * configured channel. If any {@code required} channel throws a
     * {@link ChannelException}, that exception will be propagated.
     * <p>
     * <p>Note that if multiple channels are configured, some {@link Transaction}s
     * may have already been committed while others may be rolled back in the
     * case of an exception.
     *
     * @param event The event to put into the configured channels.
     * @throws ChannelException when a write to a required channel fails.
     */
    public void processEvent(Event event) {
        event = interceptorChain.intercept(event);
        if (event == null) {
            return;
        }
        // Process required channels
        List<Channel> requiredChannels = selector.getRequiredChannels(event);
        for (Channel reqChannel : requiredChannels) {
            Transaction tx = reqChannel.getTransaction();
            Preconditions.checkNotNull(tx, "Transaction object must not be null");
            try {
                tx.begin();
                reqChannel.put(event);
                tx.commit();
            } catch (Throwable t) {
                tx.rollback();
                if (t instanceof Error) {
                    LOG.error("Error while writing to required channel: " + reqChannel, t);
                    throw (Error) t;
                } else if (t instanceof ChannelException) {
                    throw (ChannelException) t;
                } else {
                    throw new ChannelException("Unable to put event on required " + "channel: " + reqChannel, t);
                }
            } finally {
                if (tx != null) {
                    tx.close();
                }
            }
        }
        // Process optional channels
        List<Channel> optionalChannels = selector.getOptionalChannels(event);
        for (Channel optChannel : optionalChannels) {
            Transaction tx = null;
            try {
                tx = optChannel.getTransaction();
                tx.begin();
                optChannel.put(event);
                tx.commit();
            } catch (Throwable t) {
                tx.rollback();
                LOG.error("Unable to put event on optional channel: " + optChannel, t);
                if (t instanceof Error) {
                    throw (Error) t;
                }
            } finally {
                if (tx != null) {
                    tx.close();
                }
            }
        }
    }
}

17 View Complete Implementation : TestNetcatSource.java
Copyright Apache License 2.0
Author : apache
/**
 * We set up the the Netcat source and Flume Memory Channel on localhost
 *
 * @throws UnknownHostException
 */
@Before
public void setUp() throws UnknownHostException {
    localhost = InetAddress.getByName("127.0.0.1");
    source = new NetcatSource();
    channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
}

17 View Complete Implementation : ChannelProcessor.java
Copyright Apache License 2.0
Author : cloudera
/**
 * A channel processor exposes operations to put {@link Event}s into
 * {@link Channel}s. These operations will propagate a {@link ChannelException}
 * if any errors occur while attempting to write to {@code required} channels.
 *
 * Each channel processor instance is configured with a {@link ChannelSelector}
 * instance that specifies which channels are
 * {@linkplain ChannelSelector#getRequiredChannels(Event) required} and which
 * channels are
 * {@linkplain ChannelSelector#getOptionalChannels(Event) optional}.
 */
public clreplaced ChannelProcessor implements Configurable {

    private static final Logger LOG = LoggerFactory.getLogger(ChannelProcessor.clreplaced);

    private final ChannelSelector selector;

    private final InterceptorChain interceptorChain;

    public ChannelProcessor(ChannelSelector selector) {
        this.selector = selector;
        this.interceptorChain = new InterceptorChain();
    }

    public void initialize() {
        interceptorChain.initialize();
    }

    public void close() {
        interceptorChain.close();
    }

    /**
     * The Context of the replacedociated Source is preplaceded.
     * @param context
     */
    @Override
    public void configure(Context context) {
        configureInterceptors(context);
    }

    // WARNING: throws FlumeException (is that ok?)
    private void configureInterceptors(Context context) {
        List<Interceptor> interceptors = Lists.newLinkedList();
        String interceptorListStr = context.getString("interceptors", "");
        if (interceptorListStr.isEmpty()) {
            return;
        }
        String[] interceptorNames = interceptorListStr.split("\\s+");
        Context interceptorContexts = new Context(context.getSubProperties("interceptors."));
        // run through and instantiate all the interceptors specified in the Context
        for (String interceptorName : interceptorNames) {
            Context interceptorContext = new Context(interceptorContexts.getSubProperties(interceptorName + "."));
            String type = interceptorContext.getString("type");
            try {
                // TODO: support aliases for built-in interceptor types
                Clreplaced<?> clazz = Clreplaced.forName(type);
                if (Interceptor.Builder.clreplaced.isreplacedignableFrom(clazz)) {
                    Interceptor.Builder builder = (Interceptor.Builder) clazz.newInstance();
                    builder.configure(interceptorContext);
                    interceptors.add(builder.build());
                }
            } catch (ClreplacedNotFoundException e) {
                LOG.error("Builder clreplaced not found. Exception follows.", e);
                throw new FlumeException("Interceptor.Builder not found.", e);
            } catch (InstantiationException e) {
                LOG.error("Could not instantiate Builder. Exception follows.", e);
                throw new FlumeException("Interceptor.Builder not constructable.", e);
            } catch (IllegalAccessException e) {
                LOG.error("Unable to access Builder. Exception follows.", e);
                throw new FlumeException("Unable to access Interceptor.Builder.", e);
            }
        }
        interceptorChain.setInterceptors(interceptors);
    }

    public ChannelSelector getSelector() {
        return selector;
    }

    /**
     * Attempts to {@linkplain Channel#put(Event) put} the given events into each
     * configured channel. If any {@code required} channel throws a
     * {@link ChannelException}, that exception will be propagated.
     *
     * <p>Note that if multiple channels are configured, some {@link Transaction}s
     * may have already been committed while others may be rolled back in the
     * case of an exception.
     *
     * @param events A list of events to put into the configured channels.
     * @throws ChannelException when a write to a required channel fails.
     */
    public void processEventBatch(List<Event> events) {
        Preconditions.checkNotNull(events, "Event list must not be null");
        events = interceptorChain.intercept(events);
        Map<Channel, List<Event>> reqChannelQueue = new LinkedHashMap<Channel, List<Event>>();
        Map<Channel, List<Event>> optChannelQueue = new LinkedHashMap<Channel, List<Event>>();
        for (Event event : events) {
            List<Channel> reqChannels = selector.getRequiredChannels(event);
            for (Channel ch : reqChannels) {
                List<Event> eventQueue = reqChannelQueue.get(ch);
                if (eventQueue == null) {
                    eventQueue = new ArrayList<Event>();
                    reqChannelQueue.put(ch, eventQueue);
                }
                eventQueue.add(event);
            }
            List<Channel> optChannels = selector.getOptionalChannels(event);
            for (Channel ch : optChannels) {
                List<Event> eventQueue = optChannelQueue.get(ch);
                if (eventQueue == null) {
                    eventQueue = new ArrayList<Event>();
                    optChannelQueue.put(ch, eventQueue);
                }
                eventQueue.add(event);
            }
        }
        // Process required channels
        for (Channel reqChannel : reqChannelQueue.keySet()) {
            Transaction tx = null;
            try {
                tx = reqChannel.getTransaction();
                tx.begin();
                List<Event> batch = reqChannelQueue.get(reqChannel);
                for (Event event : batch) {
                    reqChannel.put(event);
                }
                tx.commit();
            } catch (ChannelException ex) {
                tx.rollback();
                throw ex;
            } finally {
                if (tx != null) {
                    tx.close();
                }
            }
        }
        // Process optional channels
        for (Channel optChannel : optChannelQueue.keySet()) {
            Transaction tx = null;
            try {
                tx = optChannel.getTransaction();
                tx.begin();
                List<Event> batch = optChannelQueue.get(optChannel);
                for (Event event : batch) {
                    optChannel.put(event);
                }
                tx.commit();
            } catch (ChannelException ex) {
                tx.rollback();
                LOG.warn("Unable to put event on optional channel", ex);
            } finally {
                if (tx != null) {
                    tx.close();
                }
            }
        }
    }

    /**
     * Attempts to {@linkplain Channel#put(Event) put} the given event into each
     * configured channel. If any {@code required} channel throws a
     * {@link ChannelException}, that exception will be propagated.
     *
     * <p>Note that if multiple channels are configured, some {@link Transaction}s
     * may have already been committed while others may be rolled back in the
     * case of an exception.
     *
     * @param event The event to put into the configured channels.
     * @throws ChannelException when a write to a required channel fails.
     */
    public void processEvent(Event event) {
        event = interceptorChain.intercept(event);
        if (event == null) {
            return;
        }
        // Process required channels
        List<Channel> requiredChannels = selector.getRequiredChannels(event);
        for (Channel requiredChannel : requiredChannels) {
            Transaction tx = requiredChannel.getTransaction();
            Preconditions.checkNotNull(tx, "Transaction object must not be null");
            try {
                tx.begin();
                requiredChannel.put(event);
                tx.commit();
            } catch (ChannelException ex) {
                tx.rollback();
                throw ex;
            } catch (Exception e) {
                tx.rollback();
                throw new ChannelException("Unexpected error", e);
            } finally {
                if (tx != null) {
                    tx.close();
                }
            }
        }
        // Process optional channels
        List<Channel> optionalChannels = selector.getOptionalChannels(event);
        for (Channel optionalChannel : optionalChannels) {
            Transaction tx = null;
            try {
                tx = optionalChannel.getTransaction();
                tx.begin();
                optionalChannel.put(event);
                tx.commit();
            } catch (ChannelException ex) {
                tx.rollback();
                LOG.warn("Unable to put event on optional channel " + optionalChannel.getName(), ex);
            } finally {
                if (tx != null) {
                    tx.close();
                }
            }
        }
    }
}

17 View Complete Implementation : TestAvroSource.java
Copyright Apache License 2.0
Author : cloudera
@Before
public void setUp() {
    source = new AvroSource();
    channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
}

17 View Complete Implementation : TestLegacyAvroSource.java
Copyright Apache License 2.0
Author : cloudera
@Before
public void setUp() {
    source = new AvroLegacySource();
    channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
}

17 View Complete Implementation : TestThriftLegacySource.java
Copyright Apache License 2.0
Author : cloudera
@Before
public void setUp() {
    source = new ThriftLegacySource();
    channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
}

17 View Complete Implementation : TestDefaultLogicalNodeManager.java
Copyright Apache License 2.0
Author : cloudera
@Test
public void testNodeStartStops() throws LifecycleException, InterruptedException {
    Set<LifecycleAware> testNodes = new HashSet<LifecycleAware>();
    for (int i = 0; i < 30; i++) {
        SequenceGeneratorSource source = new SequenceGeneratorSource();
        List<Channel> channels = new ArrayList<Channel>();
        channels.add(new MemoryChannel());
        ChannelSelector rcs = new ReplicatingChannelSelector();
        rcs.setChannels(channels);
        source.setChannelProcessor(new ChannelProcessor(rcs));
        PollableSourceRunner sourceRunner = new PollableSourceRunner();
        sourceRunner.setSource(source);
        testNodes.add(sourceRunner);
    }
    nodeManager.start();
    replacedert.replacedertTrue("Node manager didn't reach START or ERROR", LifecycleController.waitForOneOf(nodeManager, LifecycleState.START_OR_ERROR, 5000));
    for (LifecycleAware node : testNodes) {
        nodeManager.add(node);
    }
    Thread.sleep(5000);
    nodeManager.stop();
    replacedert.replacedertTrue("Node manager didn't reach STOP or ERROR", LifecycleController.waitForOneOf(nodeManager, LifecycleState.STOP_OR_ERROR, 5000));
}

17 View Complete Implementation : TestDefaultLogicalNodeManager.java
Copyright Apache License 2.0
Author : cloudera
@Test
public void testErrorNode() throws LifecycleException, InterruptedException {
    Set<LifecycleAware> testNodes = new HashSet<LifecycleAware>();
    for (int i = 0; i < 30; i++) {
        SequenceGeneratorSource source = new SequenceGeneratorSource();
        List<Channel> channels = new ArrayList<Channel>();
        channels.add(new MemoryChannel());
        ChannelSelector rcs = new ReplicatingChannelSelector();
        rcs.setChannels(channels);
        source.setChannelProcessor(new ChannelProcessor(rcs));
        PollableSourceRunner sourceRunner = new PollableSourceRunner();
        sourceRunner.setSource(source);
        testNodes.add(sourceRunner);
    }
    nodeManager.start();
    replacedert.replacedertTrue("Node manager didn't reach START or ERROR", LifecycleController.waitForOneOf(nodeManager, LifecycleState.START_OR_ERROR, 5000));
    for (LifecycleAware node : testNodes) {
        nodeManager.add(node);
    }
    Thread.sleep(5000);
    nodeManager.stop();
    replacedert.replacedertTrue("Node manager didn't reach STOP or ERROR", LifecycleController.waitForOneOf(nodeManager, LifecycleState.STOP_OR_ERROR, 5000));
}

17 View Complete Implementation : TestLoadBalancingLog4jAppender.java
Copyright Apache License 2.0
Author : javachen
public clreplaced TestLoadBalancingLog4replacedpender {

    private final List<CountingAvroSource> sources = Lists.newArrayList();

    private Channel ch;

    private ChannelSelector rcs;

    private Logger fixture;

    private boolean slowDown = false;

    @Before
    public void initiate() throws InterruptedException {
        ch = new MemoryChannel();
        configureChannel();
    }

    private void configureChannel() {
        Configurables.configure(ch, new Context());
        List<Channel> channels = new ArrayList<Channel>();
        channels.add(ch);
        rcs = new ReplicatingChannelSelector();
        rcs.setChannels(channels);
    }

    @After
    public void cleanUp() {
        for (Source source : sources) {
            source.stop();
        }
    }

    @Test
    public void testLog4replacedpenderRoundRobin() throws IOException {
        int numberOfMsgs = 1000;
        int expectedPerSource = 500;
        File TESTFILE = new File(TestLoadBalancingLog4replacedpender.clreplaced.getClreplacedLoader().getResource("flume-loadbalancinglog4jtest.properties").getFile());
        startSources(TESTFILE, false, new int[] { 25430, 25431 });
        sendAndreplacedertMessages(numberOfMsgs);
        for (CountingAvroSource source : sources) {
            replacedert.replacedertEquals(expectedPerSource, source.appendCount.get());
        }
    }

    @Test
    public void testLog4replacedpenderRandom() throws IOException {
        int numberOfMsgs = 1000;
        File TESTFILE = new File(TestLoadBalancingLog4replacedpender.clreplaced.getClreplacedLoader().getResource("flume-loadbalancing-rnd-log4jtest.properties").getFile());
        startSources(TESTFILE, false, new int[] { 25430, 25431, 25432, 25433, 25434, 25435, 25436, 25437, 25438, 25439 });
        sendAndreplacedertMessages(numberOfMsgs);
        int total = 0;
        Set<Integer> counts = new HashSet<Integer>();
        for (CountingAvroSource source : sources) {
            total += source.appendCount.intValue();
            counts.add(source.appendCount.intValue());
        }
        // We are not testing distribution this is tested in the client
        replacedert.replacedertTrue("Very unusual distribution " + counts.size(), counts.size() > 2);
        replacedert.replacedertTrue("Missing events", total == numberOfMsgs);
    }

    @Test
    public void testRandomBackoff() throws Exception {
        File TESTFILE = new File(TestLoadBalancingLog4replacedpender.clreplaced.getClreplacedLoader().getResource("flume-loadbalancing-backoff-log4jtest.properties").getFile());
        startSources(TESTFILE, false, new int[] { 25430, 25431, 25432 });
        sources.get(0).setFail();
        sources.get(2).setFail();
        sendAndreplacedertMessages(50);
        replacedert.replacedertEquals(50, sources.get(1).appendCount.intValue());
        replacedert.replacedertEquals(0, sources.get(0).appendCount.intValue());
        replacedert.replacedertEquals(0, sources.get(2).appendCount.intValue());
        sources.get(0).setOk();
        // s0 should still be backed off
        sources.get(1).setFail();
        try {
            send(1);
            // nothing should be able to process right now
            replacedert.fail("Expected EventDeliveryException");
        } catch (FlumeException e) {
            replacedert.replacedertTrue(e.getCause() instanceof EventDeliveryException);
        }
        // wait for s0 to no longer be backed off
        Thread.sleep(2500);
        sendAndreplacedertMessages(50);
        replacedert.replacedertEquals(50, sources.get(0).appendCount.intValue());
        replacedert.replacedertEquals(50, sources.get(1).appendCount.intValue());
        replacedert.replacedertEquals(0, sources.get(2).appendCount.intValue());
    }

    @Test
    public void testRandomBackoffUnsafeMode() throws Exception {
        File TESTFILE = new File(TestLoadBalancingLog4replacedpender.clreplaced.getClreplacedLoader().getResource("flume-loadbalancing-backoff-log4jtest.properties").getFile());
        startSources(TESTFILE, true, new int[] { 25430, 25431, 25432 });
        sources.get(0).setFail();
        sources.get(1).setFail();
        sources.get(2).setFail();
        sendAndreplacedertFail();
    }

    @Test(expected = EventDeliveryException.clreplaced)
    public void testTimeout() throws Throwable {
        File TESTFILE = new File(TestLoadBalancingLog4replacedpender.clreplaced.getClreplacedLoader().getResource("flume-loadbalancinglog4jtest.properties").getFile());
        ch = new TestLog4replacedpender.SlowMemoryChannel(2000);
        configureChannel();
        slowDown = true;
        startSources(TESTFILE, false, new int[] { 25430, 25431, 25432 });
        int level = 20000;
        String msg = "This is log message number" + String.valueOf(level);
        try {
            fixture.log(Level.toLevel(level), msg);
        } catch (FlumeException ex) {
            throw ex.getCause();
        }
    }

    @Test(expected = EventDeliveryException.clreplaced)
    public void testRandomBackoffNotUnsafeMode() throws Throwable {
        File TESTFILE = new File(TestLoadBalancingLog4replacedpender.clreplaced.getClreplacedLoader().getResource("flume-loadbalancing-backoff-log4jtest.properties").getFile());
        startSources(TESTFILE, false, new int[] { 25430, 25431, 25432 });
        sources.get(0).setFail();
        sources.get(1).setFail();
        sources.get(2).setFail();
        try {
            sendAndreplacedertFail();
        } catch (FlumeException ex) {
            throw ex.getCause();
        }
    }

    private void send(int numberOfMsgs) throws EventDeliveryException {
        for (int count = 0; count < numberOfMsgs; count++) {
            int level = count % 5;
            String msg = "This is log message number" + String.valueOf(count);
            fixture.log(Level.toLevel(level), msg);
        }
    }

    private void sendAndreplacedertFail() throws IOException {
        int level = 20000;
        String msg = "This is log message number" + String.valueOf(level);
        fixture.log(Level.toLevel(level), msg);
        Transaction transaction = ch.getTransaction();
        transaction.begin();
        Event event = ch.take();
        replacedert.replacedertNull(event);
        transaction.commit();
        transaction.close();
    }

    private void sendAndreplacedertMessages(int numberOfMsgs) throws IOException {
        for (int count = 0; count < numberOfMsgs; count++) {
            int level = count % 5;
            String msg = "This is log message number" + String.valueOf(count);
            fixture.log(Level.toLevel(level), msg);
            Transaction transaction = ch.getTransaction();
            transaction.begin();
            Event event = ch.take();
            replacedert.replacedertNotNull(event);
            replacedert.replacedertEquals(new String(event.getBody(), "UTF8"), msg);
            Map<String, String> hdrs = event.getHeaders();
            replacedert.replacedertNotNull(hdrs.get(Log4jAvroHeaders.TIMESTAMP.toString()));
            replacedert.replacedertEquals(Level.toLevel(level), Level.toLevel(hdrs.get(Log4jAvroHeaders.LOG_LEVEL.toString())));
            replacedert.replacedertEquals(fixture.getName(), hdrs.get(Log4jAvroHeaders.LOGGER_NAME.toString()));
            replacedert.replacedertEquals("UTF8", hdrs.get(Log4jAvroHeaders.MESSAGE_ENCODING.toString()));
            // To confirm on console we actually got the body
            System.out.println("Got body: " + new String(event.getBody(), "UTF8"));
            transaction.commit();
            transaction.close();
        }
    }

    private void startSources(File log4jProps, boolean unsafeMode, int... ports) throws IOException {
        for (int port : ports) {
            CountingAvroSource source = new CountingAvroSource(port);
            Context context = new Context();
            context.put("port", String.valueOf(port));
            context.put("bind", "0.0.0.0");
            Configurables.configure(source, context);
            sources.add(source);
            source.setChannelProcessor(new ChannelProcessor(rcs));
        }
        for (Source source : sources) {
            source.start();
        }
        // The properties file having Avro port info should be loaded only
        // after the test begins, else log4j tries to connect to the source
        // before the source has started up in the above function, since
        // log4j setup is completed before the @Before calls also.
        // This will cause the test to fail even before it starts!
        FileReader reader = new FileReader(log4jProps);
        Properties props = new Properties();
        props.load(reader);
        props.setProperty("log4j.appender.out2.UnsafeMode", String.valueOf(unsafeMode));
        if (slowDown) {
            props.setProperty("log4j.appender.out2.Timeout", String.valueOf(1000));
        }
        PropertyConfigurator.configure(props);
        fixture = LogManager.getLogger(TestLoadBalancingLog4replacedpender.clreplaced);
    }

    static clreplaced CountingAvroSource extends AvroSource {

        AtomicInteger appendCount = new AtomicInteger();

        volatile boolean isFail = false;

        private final int port2;

        public CountingAvroSource(int port) {
            port2 = port;
        }

        public void setOk() {
            this.isFail = false;
        }

        public void setFail() {
            this.isFail = true;
        }

        @Override
        public String getName() {
            return "testing..." + port2;
        }

        @Override
        public Status append(AvroFlumeEvent avroEvent) {
            if (isFail) {
                return Status.FAILED;
            }
            appendCount.incrementAndGet();
            return super.append(avroEvent);
        }

        @Override
        public Status appendBatch(List<AvroFlumeEvent> events) {
            if (isFail) {
                return Status.FAILED;
            }
            appendCount.addAndGet(events.size());
            return super.appendBatch(events);
        }
    }
}

16 View Complete Implementation : FlumeThriftService.java
Copyright Apache License 2.0
Author : apache
public static void main(String[] args) throws Exception {
    // Flume Source
    ThriftSource source = new ThriftSource();
    Channel ch = new MemoryChannel();
    Configurables.configure(ch, new Context());
    Context context = new Context();
    context.put("port", String.valueOf(port));
    context.put("bind", hostname);
    Configurables.configure(source, context);
    List<Channel> channels = new ArrayList<>();
    channels.add(ch);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
    source.start();
    System.out.println("ThriftSource service start.");
    while (true) {
        Transaction transaction = ch.getTransaction();
        transaction.begin();
        Event event = ch.take();
        if (null != event) {
            System.out.println(event);
            System.out.println(new String(event.getBody()).trim());
        }
        transaction.commit();
        transaction.close();
    }
}

16 View Complete Implementation : ChannelSelectorFactory.java
Copyright Apache License 2.0
Author : apache
public static ChannelSelector create(List<Channel> channels, Map<String, String> config) {
    ChannelSelector selector = getSelectorForType(config.get(BasicConfigurationConstants.CONFIG_TYPE));
    selector.setChannels(channels);
    Context context = new Context();
    context.putAll(config);
    Configurables.configure(selector, context);
    return selector;
}

16 View Complete Implementation : TestSyslogTcpSource.java
Copyright Apache License 2.0
Author : apache
private void init(String keepFields, Context context) {
    source = new SyslogTcpSource();
    channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    List<Channel> channels = new ArrayList<>();
    channels.add(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
    context.put("host", InetAddress.getLoopbackAddress().getHostAddress());
    context.put("port", String.valueOf(TEST_SYSLOG_PORT));
    context.put("keepFields", keepFields);
    source.configure(context);
}

16 View Complete Implementation : TestSyslogUdpSource.java
Copyright Apache License 2.0
Author : apache
private void init(String keepFields, Context context) {
    source = new SyslogUDPSource();
    channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
    context.put("host", InetAddress.getLoopbackAddress().getHostAddress());
    context.put("port", String.valueOf(TEST_SYSLOG_PORT));
    context.put("keepFields", keepFields);
    source.configure(context);
}

16 View Complete Implementation : TestLegacyAvroSource.java
Copyright Apache License 2.0
Author : apache
@Before
public void setUp() throws Exception {
    source = new AvroLegacySource();
    channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
    try (ServerSocket socket = new ServerSocket(0)) {
        selectedPort = socket.getLocalPort();
    }
}

16 View Complete Implementation : TestThriftLegacySource.java
Copyright Apache License 2.0
Author : apache
@Before
public void setUp() throws Exception {
    source = new ThriftLegacySource();
    channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
    try (ServerSocket socket = new ServerSocket(0)) {
        selectedPort = socket.getLocalPort();
    }
}

16 View Complete Implementation : TestHDFSEventSinkDeadlock.java
Copyright Apache License 2.0
Author : apache
public static void main(String... args) {
    HDFSEventSink sink = new HDFSEventSink();
    sink.setName("HDFSEventSink");
    Context context = new Context(ImmutableMap.of("hdfs.path", "file:///tmp/flume-test/bucket-%t", "hdfs.filePrefix", "flumetest", "hdfs.rollInterval", "1", "hdfs.maxOpenFiles", "1", "hdfs.useLocalTimeStamp", "true"));
    Configurables.configure(sink, context);
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    final SequenceGeneratorSource source = new SequenceGeneratorSource();
    Configurables.configure(source, new Context());
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(Collections.singletonList(channel));
    source.setChannelProcessor(new ChannelProcessor(rcs));
    sink.setChannel(channel);
    channel.start();
    source.start();
    SinkProcessor sinkProcessor = new DefaultSinkProcessor();
    sinkProcessor.setSinks(Collections.singletonList(sink));
    SinkRunner sinkRunner = new SinkRunner();
    sinkRunner.setSink(sinkProcessor);
    sinkRunner.start();
    ScheduledExecutorService executor = Executors.newScheduledThreadPool(3);
    executor.execute(new Runnable() {

        @Override
        public void run() {
            int i = 0;
            while (true) {
                try {
                    source.process();
                    System.out.println(i++);
                    if (i == 250) {
                        System.out.println("No deadlock found after 250 iterations, exiting");
                        System.exit(0);
                    }
                    Thread.sleep((long) (Math.random() * 100 + 950));
                } catch (Exception e) {
                // 
                }
            }
        }
    });
    executor.scheduleAtFixedRate(new Runnable() {

        @Override
        public void run() {
            ThreadMXBean bean = ManagementFactory.getThreadMXBean();
            long[] threadIds = bean.findDeadlockedThreads();
            if (threadIds != null) {
                System.out.println("Deadlocked threads found");
                printThreadStackTraces(threadIds);
                System.exit(1);
            }
        }
    }, 0, 1, TimeUnit.SECONDS);
}

16 View Complete Implementation : FlumeAppenderTest.java
Copyright Apache License 2.0
Author : apache
@Before
public void setUp() throws Exception {
    eventSource = new AvroSource();
    channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    avroLogger = (Logger) LogManager.getLogger("avrologger");
    /*
         * Clear out all other appenders replacedociated with this logger to ensure
         * we're only hitting the Avro appender.
         */
    removeAppenders(avroLogger);
    final Context context = new Context();
    testPort = String.valueOf(AvailablePortFinder.getNextAvailable());
    context.put("port", testPort);
    context.put("bind", "0.0.0.0");
    Configurables.configure(eventSource, context);
    final List<Channel> channels = new ArrayList<>();
    channels.add(channel);
    final ChannelSelector cs = new ReplicatingChannelSelector();
    cs.setChannels(channels);
    eventSource.setChannelProcessor(new ChannelProcessor(cs));
    eventSource.start();
    replacedert.replacedertTrue("Reached start or error", LifecycleController.waitForOneOf(eventSource, LifecycleState.START_OR_ERROR));
    replacedert.replacedertEquals("Server is started", LifecycleState.START, eventSource.getLifecycleState());
}

16 View Complete Implementation : TestSyslogUdpSource.java
Copyright Apache License 2.0
Author : cloudera
@Before
public void setUp() {
    // SyslogTcpSource();
    source = new SyslogUDPSource();
    channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
    Context context = new Context();
    context.put("port", String.valueOf(TEST_SYSLOG_PORT));
    source.configure(context);
}

16 View Complete Implementation : TestLog4jAppender.java
Copyright Apache License 2.0
Author : javachen
private void configureSource() {
    List<Channel> channels = new ArrayList<Channel>();
    channels.add(ch);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
    source.start();
}

15 View Complete Implementation : TestPollableSourceRunner.java
Copyright Apache License 2.0
Author : apache
@Test
public void testLifecycle() throws InterruptedException {
    final Channel channel = new MemoryChannel();
    final CountDownLatch latch = new CountDownLatch(50);
    Configurables.configure(channel, new Context());
    final ChannelSelector cs = new ReplicatingChannelSelector();
    cs.setChannels(Lists.newArrayList(channel));
    PollableSource source = new PollableSource() {

        private String name;

        private ChannelProcessor cp = new ChannelProcessor(cs);

        @Override
        public Status process() throws EventDeliveryException {
            Transaction transaction = channel.getTransaction();
            try {
                transaction.begin();
                Event event = EventBuilder.withBody(String.valueOf("Event " + latch.getCount()).getBytes());
                latch.countDown();
                if (latch.getCount() % 20 == 0) {
                    throw new EventDeliveryException("I don't like event:" + event);
                }
                channel.put(event);
                transaction.commit();
                return Status.READY;
            } catch (EventDeliveryException e) {
                logger.error("Unable to deliver event. Exception follows.", e);
                transaction.rollback();
                return Status.BACKOFF;
            } finally {
                transaction.close();
            }
        }

        @Override
        public long getBackOffSleepIncrement() {
            return PollableSourceConstants.DEFAULT_BACKOFF_SLEEP_INCREMENT;
        }

        @Override
        public long getMaxBackOffSleepInterval() {
            return PollableSourceConstants.DEFAULT_MAX_BACKOFF_SLEEP;
        }

        @Override
        public void start() {
        // Unused.
        }

        @Override
        public void stop() {
        // Unused.
        }

        @Override
        public LifecycleState getLifecycleState() {
            // Unused.
            return null;
        }

        @Override
        public void setName(String name) {
            this.name = name;
        }

        @Override
        public String getName() {
            return name;
        }

        @Override
        public void setChannelProcessor(ChannelProcessor channelProcessor) {
            cp = channelProcessor;
        }

        @Override
        public ChannelProcessor getChannelProcessor() {
            return cp;
        }
    };
    sourceRunner.setSource(source);
    sourceRunner.start();
    latch.await();
    sourceRunner.stop();
}

15 View Complete Implementation : TestNetcatSource.java
Copyright Apache License 2.0
Author : apache
@Before
public void setUp() {
    logger.info("Running setup");
    channel = new MemoryChannel();
    source = new NetcatSource();
    Context context = new Context();
    Configurables.configure(channel, context);
    List<Channel> channels = Lists.newArrayList(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
}

15 View Complete Implementation : TestPollableSourceRunner.java
Copyright Apache License 2.0
Author : cloudera
@Test
public void testLifecycle() throws InterruptedException {
    final Channel channel = new MemoryChannel();
    final CountDownLatch latch = new CountDownLatch(50);
    Configurables.configure(channel, new Context());
    final ChannelSelector cs = new ReplicatingChannelSelector();
    cs.setChannels(Lists.newArrayList(channel));
    PollableSource source = new PollableSource() {

        private String name;

        private ChannelProcessor cp = new ChannelProcessor(cs);

        @Override
        public Status process() throws EventDeliveryException {
            Transaction transaction = channel.getTransaction();
            try {
                transaction.begin();
                Event event = EventBuilder.withBody(String.valueOf("Event " + latch.getCount()).getBytes());
                latch.countDown();
                if (latch.getCount() % 20 == 0) {
                    throw new EventDeliveryException("I don't like event:" + event);
                }
                channel.put(event);
                transaction.commit();
                return Status.READY;
            } catch (EventDeliveryException e) {
                logger.error("Unable to deliver event. Exception follows.", e);
                transaction.rollback();
                return Status.BACKOFF;
            } finally {
                transaction.close();
            }
        }

        @Override
        public void start() {
        // Unused.
        }

        @Override
        public void stop() {
        // Unused.
        }

        @Override
        public LifecycleState getLifecycleState() {
            // Unused.
            return null;
        }

        @Override
        public void setName(String name) {
            this.name = name;
        }

        @Override
        public String getName() {
            return name;
        }

        @Override
        public void setChannelProcessor(ChannelProcessor channelProcessor) {
            cp = channelProcessor;
        }

        @Override
        public ChannelProcessor getChannelProcessor() {
            return cp;
        }
    };
    sourceRunner.setSource(source);
    sourceRunner.start();
    latch.await();
    sourceRunner.stop();
}

15 View Complete Implementation : TestHTTPSource.java
Copyright Apache License 2.0
Author : javachen
@BeforeClreplaced
public static void setUpClreplaced() throws Exception {
    selectedPort = findFreePort();
    source = new HTTPSource();
    channel = new MemoryChannel();
    Context ctx = new Context();
    ctx.put("capacity", "100");
    Configurables.configure(channel, ctx);
    List<Channel> channels = new ArrayList<Channel>(1);
    channels.add(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
    channel.start();
    Context context = new Context();
    context.put("port", String.valueOf(selectedPort));
    context.put("host", "0.0.0.0");
    Configurables.configure(source, context);
    source.start();
}

15 View Complete Implementation : TestLog4jAppender.java
Copyright Apache License 2.0
Author : kite-sdk
@Before
public void initiate() throws Exception {
    int port = 25430;
    source = new AvroSource();
    ch = new MemoryChannel();
    Configurables.configure(ch, new Context());
    Context context = new Context();
    context.put("port", String.valueOf(port));
    context.put("bind", "localhost");
    Configurables.configure(source, context);
    List<Channel> channels = new ArrayList<Channel>();
    channels.add(ch);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
    source.start();
    File TESTFILE = new File(TestLog4replacedpender.clreplaced.getClreplacedLoader().getResource("flume-log4jtest.properties").getFile());
    FileReader reader = new FileReader(TESTFILE);
    props = new Properties();
    props.load(reader);
    reader.close();
}

15 View Complete Implementation : TwitterSourceTest.java
Copyright GNU Affero General Public License v3.0
Author : telefonicaid
// From flume v1.7.0
@Test
public void testBasic() throws Exception {
    System.out.println(getTestTraceHead("[TwitterSourceTest.basic]") + "-------- Start source.");
    Context context = new Context();
    context.put("consumerKey", consumerKey);
    context.put("consumerSecret", consumerSecret);
    context.put("accessToken", accessToken);
    context.put("accessTokenSecret", accessTokenSecret);
    context.put("maxBatchDurationMillis", "1000");
    TwitterSource source = new TwitterSource();
    source.configure(context);
    Map<String, String> channelContext = new HashMap();
    channelContext.put("capacity", "1000000");
    // for faster tests
    channelContext.put("keep-alive", "0");
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context(channelContext));
    Sink sink = new LoggerSink();
    sink.setChannel(channel);
    sink.start();
    DefaultSinkProcessor proc = new DefaultSinkProcessor();
    proc.setSinks(Collections.singletonList(sink));
    SinkRunner sinkRunner = new SinkRunner(proc);
    sinkRunner.start();
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(Collections.singletonList(channel));
    ChannelProcessor chp = new ChannelProcessor(rcs);
    source.setChannelProcessor(chp);
    try {
        source.start();
        Thread.sleep(500);
        source.stop();
        System.out.println(getTestTraceHead("[TwitterSourceTest.basic]") + "-  OK  - Twitter source started properly.");
    } catch (replacedertionError e) {
        System.out.println(getTestTraceHead("[TwitterSourceTest.basic]") + "- FAIL - Twitter source could not start.");
        throw e;
    }
    // try catch
    sinkRunner.stop();
    sink.stop();
}

14 View Complete Implementation : TestLog4jAppenderWithAvro.java
Copyright Apache License 2.0
Author : apache
@Before
public void setUp() throws Exception {
    URL schemaUrl = getClreplaced().getClreplacedLoader().getResource("myrecord.avsc");
    Files.copy(Resources.newInputStreamSupplier(schemaUrl), new File("/tmp/myrecord.avsc"));
    port = getFreePort();
    source = new AvroSource();
    ch = new MemoryChannel();
    Configurables.configure(ch, new Context());
    Context context = new Context();
    context.put("port", String.valueOf(port));
    context.put("bind", "localhost");
    Configurables.configure(source, context);
    List<Channel> channels = new ArrayList<>();
    channels.add(ch);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
    source.start();
}