org.apache.hadoop.hbase.util.Pair - java examples

Here are the examples of the java api org.apache.hadoop.hbase.util.Pair taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

155 Examples 7

19 View Complete Implementation : MapReduceBackupMergeJob.java
Copyright Apache License 2.0
Author : apache
protected List<Path> toPathList(List<Pair<TableName, Path>> processedTableList) {
    ArrayList<Path> list = new ArrayList<>();
    for (Pair<TableName, Path> p : processedTableList) {
        list.add(p.getSecond());
    }
    return list;
}

19 View Complete Implementation : StripeCompactionPolicy.java
Copyright Apache License 2.0
Author : apache
public StripeCompactionRequest createEmptyRequest(StripeInformationProvider si, CompactionRequestImpl request) {
    // Treat as L0-ish compaction with fixed set of files, and hope for the best.
    if (si.getStripeCount() > 0) {
        return new BoundaryStripeCompactionRequest(request, si.getStripeBoundaries());
    }
    Pair<Long, Integer> targetKvsAndCount = estimateTargetKvs(request.getFiles(), this.config.getInitialCount());
    return new SplitStripeCompactionRequest(request, OPEN_KEY, OPEN_KEY, targetKvsAndCount.getSecond(), targetKvsAndCount.getFirst());
}

19 View Complete Implementation : MapReduceBackupMergeJob.java
Copyright Apache License 2.0
Author : apache
protected List<TableName> toTableNameList(List<Pair<TableName, Path>> processedTableList) {
    ArrayList<TableName> list = new ArrayList<>();
    for (Pair<TableName, Path> p : processedTableList) {
        list.add(p.getFirst());
    }
    return list;
}

19 View Complete Implementation : TestRSGroupsAdmin2.java
Copyright Apache License 2.0
Author : apache
@Test
public void testFailedMoveTablesAndRepair() throws Exception {
    // This UT calls moveTables() twice to test the idempotency of it.
    // The first time, movement fails because a region is made in SPLITTING state
    // which will not be moved.
    // The second time, the region state is OPEN and check if all
    // regions on target group servers after the call.
    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
    Iterator iterator = newGroup.getServers().iterator();
    Address newGroupServer1 = (Address) iterator.next();
    // create table
    // randomly set a region state to SPLITTING to make move abort
    Pair<ServerName, RegionStateNode> gotPair = createTableWithRegionSplitting(newGroup, new Random().nextInt(8) + 4);
    RegionStateNode rsn = gotPair.getSecond();
    // move table to newGroup and check regions
    try {
        rsGroupAdmin.moveTables(Sets.newHashSet(tableName), newGroup.getName());
        fail("should get IOException when retry exhausted but there still exists failed moved " + "regions");
    } catch (Exception e) {
        replacedertTrue(e.getMessage().contains(gotPair.getSecond().getRegionInfo().getRegionNamereplacedtring()));
    }
    for (RegionInfo regionInfo : master.getreplacedignmentManager().getreplacedignedRegions()) {
        if (regionInfo.getTable().equals(tableName) && regionInfo.equals(rsn.getRegionInfo())) {
            replacedertNotEquals(master.getreplacedignmentManager().getRegionStates().getRegionServerOfRegion(regionInfo).getAddress(), newGroupServer1);
        }
    }
    // retry move table to newGroup and check if all regions are corrected
    rsn.setState(RegionState.State.OPEN);
    rsGroupAdmin.moveTables(Sets.newHashSet(tableName), newGroup.getName());
    for (RegionInfo regionInfo : master.getreplacedignmentManager().getreplacedignedRegions()) {
        if (regionInfo.getTable().equals(tableName)) {
            replacedertEquals(master.getreplacedignmentManager().getRegionStates().getRegionServerOfRegion(regionInfo).getAddress(), newGroupServer1);
        }
    }
}

19 View Complete Implementation : TestRSGroupsAdmin2.java
Copyright Apache License 2.0
Author : apache
@Test
public void testFailedMoveServersAndRepair() throws Exception {
    // This UT calls moveServers() twice to test the idempotency of it.
    // The first time, movement fails because a region is made in SPLITTING state
    // which will not be moved.
    // The second time, the region state is OPEN and check if all
    // regions on target group servers after the call.
    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
    // create table
    // randomly set a region state to SPLITTING to make move abort
    Pair<ServerName, RegionStateNode> gotPair = createTableWithRegionSplitting(newGroup, new Random().nextInt(8) + 4);
    RegionStateNode rsn = gotPair.getSecond();
    ServerName srcServer = rsn.getRegionLocation();
    // move server to newGroup and check regions
    try {
        rsGroupAdmin.moveServers(Sets.newHashSet(srcServer.getAddress()), newGroup.getName());
        fail("should get IOException when retry exhausted but there still exists failed moved " + "regions");
    } catch (Exception e) {
        replacedertTrue(e.getMessage().contains(gotPair.getSecond().getRegionInfo().getRegionNamereplacedtring()));
    }
    for (RegionInfo regionInfo : master.getreplacedignmentManager().getreplacedignedRegions()) {
        if (regionInfo.getTable().equals(tableName) && regionInfo.equals(rsn.getRegionInfo())) {
            replacedertEquals(master.getreplacedignmentManager().getRegionStates().getRegionServerOfRegion(regionInfo), srcServer);
        }
    }
    // retry move server to newGroup and check if all regions on srcServer was moved
    rsn.setState(RegionState.State.OPEN);
    rsGroupAdmin.moveServers(Sets.newHashSet(srcServer.getAddress()), newGroup.getName());
    replacedertEquals(master.getreplacedignmentManager().getRegionsOnServer(srcServer).size(), 0);
}

19 View Complete Implementation : TestRSGroupsAdmin2.java
Copyright Apache License 2.0
Author : apache
@Test
public void testFailedMoveBeforeRetryExhaustedWhenMoveServer() throws Exception {
    String groupName = getGroupName(name.getMethodName());
    rsGroupAdmin.addRSGroup(groupName);
    final RSGroupInfo newGroup = rsGroupAdmin.getRSGroupInfo(groupName);
    Pair<ServerName, RegionStateNode> gotPair = createTableWithRegionSplitting(newGroup, 10);
    // start thread to recover region state
    final ServerName movedServer = gotPair.getFirst();
    final RegionStateNode rsn = gotPair.getSecond();
    AtomicBoolean changed = new AtomicBoolean(false);
    Thread t1 = recoverRegionStateThread(movedServer, server -> master.getreplacedignmentManager().getRegionsOnServer(movedServer), rsn, changed);
    t1.start();
    // move target server to group
    Thread t2 = new Thread(() -> {
        LOG.info("thread2 start running, to move regions");
        try {
            rsGroupAdmin.moveServers(Sets.newHashSet(movedServer.getAddress()), newGroup.getName());
        } catch (IOException e) {
            LOG.error("move server error", e);
        }
    });
    t2.start();
    t1.join();
    t2.join();
    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {

        @Override
        public boolean evaluate() {
            if (changed.get()) {
                return master.getreplacedignmentManager().getRegionsOnServer(movedServer).size() == 0 && !rsn.getRegionLocation().equals(movedServer);
            }
            return false;
        }
    });
}

19 View Complete Implementation : TestReversibleScanners.java
Copyright Apache License 2.0
Author : apache
private Pair<Integer, Integer> getNextReadableNumWithBackwardScan(int startRowNum, int startQualNum, int readPoint) {
    Pair<Integer, Integer> nextReadableNum = null;
    boolean findExpected = false;
    for (int i = startRowNum; i >= 0; i--) {
        for (int j = (i == startRowNum ? startQualNum : 0); j < QUALSIZE; j++) {
            if (makeMVCC(i, j) <= readPoint) {
                nextReadableNum = new Pair<>(i, j);
                findExpected = true;
                break;
            }
        }
        if (findExpected)
            break;
    }
    return nextReadableNum;
}

19 View Complete Implementation : TestMetaTableAccessor.java
Copyright Apache License 2.0
Author : apache
private static List<RegionInfo> testGettingTableRegions(final Connection connection, final TableName name, final int regionCount) throws IOException, InterruptedException {
    List<RegionInfo> regions = MetaTableAccessor.getTableRegions(connection, name);
    replacedertEquals(regionCount, regions.size());
    Pair<RegionInfo, ServerName> pair = MetaTableAccessor.getRegion(connection, regions.get(0).getRegionName());
    replacedertEquals(regions.get(0).getEncodedName(), pair.getFirst().getEncodedName());
    return regions;
}

19 View Complete Implementation : AggregationClient.java
Copyright Apache License 2.0
Author : apache
/**
 * This is the client side interface/handle for calling the std method for a
 * given cf-cq combination. It was necessary to add one more call stack as its
 * return type should be a decimal value, irrespective of what
 * columninterpreter says. So, this methods collects the necessary parameters
 * to compute the std and returns the double value.
 * @param table table to scan.
 * @param ci the user's ColumnInterpreter implementation
 * @param scan the HBase scan object to use to read data from HBase
 * @return <R, S>
 * @throws Throwable The caller is supposed to handle the exception as they are thrown
 *           & propagated to it.
 */
public <R, S, P extends Message, Q extends Message, T extends Message> double std(final Table table, ColumnInterpreter<R, S, P, Q, T> ci, Scan scan) throws Throwable {
    Pair<List<S>, Long> p = getStdArgs(table, ci, scan);
    double res = 0d;
    double avg = ci.divideForAvg(p.getFirst().get(0), p.getSecond());
    double avgOfSumSq = ci.divideForAvg(p.getFirst().get(1), p.getSecond());
    // variance
    res = avgOfSumSq - (avg) * (avg);
    res = Math.pow(res, 0.5);
    return res;
}

19 View Complete Implementation : TestReversibleScanners.java
Copyright Apache License 2.0
Author : apache
private KeyValue getNextReadableKeyValueWithBackwardScan(int startRowNum, int startQualNum, int readPoint) {
    Pair<Integer, Integer> nextReadableNum = getNextReadableNumWithBackwardScan(startRowNum, startQualNum, readPoint);
    if (nextReadableNum == null)
        return null;
    return makeKV(nextReadableNum.getFirst(), nextReadableNum.getSecond());
}

19 View Complete Implementation : PolicyBasedChaosMonkey.java
Copyright Apache License 2.0
Author : apache
/**
 * Selects a random item from the given items with weights
 */
public static <T> T selectWeightedRandomItem(List<Pair<T, Integer>> items) {
    int totalWeight = 0;
    for (Pair<T, Integer> pair : items) {
        totalWeight += pair.getSecond();
    }
    int cutoff = RandomUtils.nextInt(0, totalWeight);
    int replacedmulative = 0;
    T item = null;
    // warn: O(n)
    for (int i = 0; i < items.size(); i++) {
        int curWeight = items.get(i).getSecond();
        if (cutoff < replacedmulative + curWeight) {
            item = items.get(i).getFirst();
            break;
        }
        replacedmulative += curWeight;
    }
    return item;
}

18 View Complete Implementation : ThriftConnection.java
Copyright Apache License 2.0
Author : apache
/**
 * Get a ThriftAdmin, ThriftAdmin is NOT thread safe
 * @return a ThriftAdmin
 * @throws IOException IOException
 */
@Override
public Admin getAdmin() throws IOException {
    Pair<THBaseService.Client, TTransport> client = clientBuilder.getClient();
    return new ThriftAdmin(client.getFirst(), client.getSecond(), conf);
}

18 View Complete Implementation : AggregationClient.java
Copyright Apache License 2.0
Author : apache
public synchronized Pair<NavigableMap<byte[], List<S>>, List<S>> getMedianParams() {
    List<S> l = new ArrayList<>(2);
    l.add(sumVal);
    l.add(sumWeights);
    Pair<NavigableMap<byte[], List<S>>, List<S>> p = new Pair<>(map, l);
    return p;
}

18 View Complete Implementation : VisibilityController.java
Copyright Apache License 2.0
Author : apache
@Override
public List<Pair<Cell, Cell>> postAppendBeforeWAL(ObserverContext<RegionCoprocessorEnvironment> ctx, Mutation mutation, List<Pair<Cell, Cell>> cellPairs) throws IOException {
    List<Pair<Cell, Cell>> resultPairs = new ArrayList<>(cellPairs.size());
    for (Pair<Cell, Cell> pair : cellPairs) {
        resultPairs.add(new Pair<>(pair.getFirst(), createNewCellWithTags(mutation, pair.getSecond())));
    }
    return resultPairs;
}

18 View Complete Implementation : ReplicationSink.java
Copyright Apache License 2.0
Author : apache
private void addNewTableEntryInMap(final Map<String, List<Pair<byte[], List<String>>>> bulkLoadHFileMap, byte[] family, String pathToHfileFromNS, String tableName) {
    List<String> hfilePaths = new ArrayList<>(1);
    hfilePaths.add(pathToHfileFromNS);
    Pair<byte[], List<String>> newFamilyHFilePathsPair = new Pair<>(family, hfilePaths);
    List<Pair<byte[], List<String>>> newFamilyHFilePathsList = new ArrayList<>();
    newFamilyHFilePathsList.add(newFamilyHFilePathsPair);
    bulkLoadHFileMap.put(tableName, newFamilyHFilePathsList);
}

18 View Complete Implementation : AggregationClient.java
Copyright Apache License 2.0
Author : apache
public synchronized Pair<List<S>, Long> getStdParams() {
    List<S> l = new ArrayList<>(2);
    l.add(sumVal);
    l.add(sumSqVal);
    Pair<List<S>, Long> p = new Pair<>(l, rowCountVal);
    return p;
}

18 View Complete Implementation : TestMetaTableAccessor.java
Copyright Apache License 2.0
Author : apache
private static void testGetRegion(final Connection connection, final RegionInfo region) throws IOException, InterruptedException {
    Pair<RegionInfo, ServerName> pair = MetaTableAccessor.getRegion(connection, region.getRegionName());
    replacedertEquals(region.getEncodedName(), pair.getFirst().getEncodedName());
}

18 View Complete Implementation : DefaultNettyEventLoopConfig.java
Copyright Apache License 2.0
Author : apache
/**
 * The default netty event loop config
 */
@InterfaceAudience.Private
clreplaced DefaultNettyEventLoopConfig {

    public static final Pair<EventLoopGroup, Clreplaced<? extends Channel>> GROUP_AND_CHANNEL_CLreplaced = Pair.<EventLoopGroup, Clreplaced<? extends Channel>>newPair(new NioEventLoopGroup(0, new DefaultThreadFactory("Default-IPC-NioEventLoopGroup", true, Thread.MAX_PRIORITY)), NioSocketChannel.clreplaced);
}

18 View Complete Implementation : TestFuzzyRowFilterEndToEnd.java
Copyright Apache License 2.0
Author : apache
@SuppressWarnings("unchecked")
private void runTest(Table hTable, int expectedSize) throws IOException {
    // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
    byte[] fuzzyKey1 = new byte[10];
    ByteBuffer buf = ByteBuffer.wrap(fuzzyKey1);
    buf.clear();
    buf.putShort((short) 2);
    for (int i = 0; i < 4; i++) buf.put(fuzzyValue);
    buf.putInt((short) 1);
    byte[] mask1 = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
    byte[] fuzzyKey2 = new byte[10];
    buf = ByteBuffer.wrap(fuzzyKey2);
    buf.clear();
    buf.putShort((short) 2);
    buf.putInt((short) 2);
    for (int i = 0; i < 4; i++) buf.put(fuzzyValue);
    byte[] mask2 = new byte[] { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1 };
    Pair<byte[], byte[]> pair1 = new Pair<>(fuzzyKey1, mask1);
    Pair<byte[], byte[]> pair2 = new Pair<>(fuzzyKey2, mask2);
    FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(Lists.newArrayList(pair1));
    FuzzyRowFilter fuzzyRowFilter2 = new FuzzyRowFilter(Lists.newArrayList(pair2));
    // regular test - we expect 1 row back (5 KVs)
    runScanner(hTable, expectedSize, fuzzyRowFilter1, fuzzyRowFilter2);
}

18 View Complete Implementation : TestRSGroupsAdmin2.java
Copyright Apache License 2.0
Author : apache
@Test
public void testFailedMoveServersTablesAndRepair() throws Exception {
    // This UT calls moveTablesAndServers() twice to test the idempotency of it.
    // The first time, movement fails because a region is made in SPLITTING state
    // which will not be moved.
    // The second time, the region state is OPEN and check if all
    // regions on target group servers after the call.
    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
    // create table
    final byte[] familyNameBytes = Bytes.toBytes("f");
    TableName table1 = TableName.valueOf(tableName.getNamereplacedtring() + "_1");
    TableName table2 = TableName.valueOf(tableName.getNamereplacedtring() + "_2");
    TEST_UTIL.createMultiRegionTable(table1, familyNameBytes, new Random().nextInt(12) + 4);
    TEST_UTIL.createMultiRegionTable(table2, familyNameBytes, new Random().nextInt(12) + 4);
    // randomly set a region state to SPLITTING to make move abort
    Pair<ServerName, RegionStateNode> gotPair = randomlySetRegionState(newGroup, RegionState.State.SPLITTING, table1, table2);
    RegionStateNode rsn = gotPair.getSecond();
    ServerName srcServer = rsn.getRegionLocation();
    // move server and table to newGroup and check regions
    try {
        rsGroupAdmin.moveServersAndTables(Sets.newHashSet(srcServer.getAddress()), Sets.newHashSet(table2), newGroup.getName());
        fail("should get IOException when retry exhausted but there still exists failed moved " + "regions");
    } catch (Exception e) {
        replacedertTrue(e.getMessage().contains(gotPair.getSecond().getRegionInfo().getRegionNamereplacedtring()));
    }
    for (RegionInfo regionInfo : master.getreplacedignmentManager().getreplacedignedRegions()) {
        if (regionInfo.getTable().equals(table1) && regionInfo.equals(rsn.getRegionInfo())) {
            replacedertEquals(master.getreplacedignmentManager().getRegionStates().getRegionServerOfRegion(regionInfo), srcServer);
        }
    }
    // retry moveServersAndTables to newGroup and check if all regions on srcServer belongs to
    // table2
    rsn.setState(RegionState.State.OPEN);
    rsGroupAdmin.moveServersAndTables(Sets.newHashSet(srcServer.getAddress()), Sets.newHashSet(table2), newGroup.getName());
    for (RegionInfo regionsInfo : master.getreplacedignmentManager().getRegionsOnServer(srcServer)) {
        replacedertEquals(regionsInfo.getTable(), table2);
    }
}

18 View Complete Implementation : TestMetaTableAccessor.java
Copyright Apache License 2.0
Author : apache
@Test
public void testGetRegion() throws IOException, InterruptedException {
    final String name = this.name.getMethodName();
    LOG.info("Started " + name);
    // Test get on non-existent region.
    Pair<RegionInfo, ServerName> pair = MetaTableAccessor.getRegion(connection, Bytes.toBytes("nonexistent-region"));
    replacedertNull(pair);
    LOG.info("Finished " + name);
}

18 View Complete Implementation : FuzzyRowFilter.java
Copyright Apache License 2.0
Author : apache
/**
 * @return true if and only if the fields of the filter that are serialized are equal to the
 *         corresponding fields in other. Used for testing.
 */
@Override
boolean areSerializedFieldsEqual(Filter o) {
    if (o == this)
        return true;
    if (!(o instanceof FuzzyRowFilter))
        return false;
    FuzzyRowFilter other = (FuzzyRowFilter) o;
    if (this.fuzzyKeysData.size() != other.fuzzyKeysData.size())
        return false;
    for (int i = 0; i < fuzzyKeysData.size(); ++i) {
        Pair<byte[], byte[]> thisData = this.fuzzyKeysData.get(i);
        Pair<byte[], byte[]> otherData = other.fuzzyKeysData.get(i);
        if (!(Bytes.equals(thisData.getFirst(), otherData.getFirst()) && Bytes.equals(thisData.getSecond(), otherData.getSecond()))) {
            return false;
        }
    }
    return true;
}

18 View Complete Implementation : TestRSGroupsAdmin2.java
Copyright Apache License 2.0
Author : apache
@Test
public void testFailedMoveBeforeRetryExhaustedWhenMoveTable() throws Exception {
    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
    Pair<ServerName, RegionStateNode> gotPair = createTableWithRegionSplitting(newGroup, 5);
    // move table to group
    Thread t2 = new Thread(() -> {
        LOG.info("thread2 start running, to move regions");
        try {
            rsGroupAdmin.moveTables(Sets.newHashSet(tableName), newGroup.getName());
        } catch (IOException e) {
            LOG.error("move server error", e);
        }
    });
    t2.start();
    // start thread to recover region state
    final ServerName ss = gotPair.getFirst();
    final RegionStateNode rsn = gotPair.getSecond();
    AtomicBoolean changed = new AtomicBoolean(false);
    Thread t1 = recoverRegionStateThread(ss, server -> {
        List<RegionInfo> regions = master.getreplacedignmentManager().getRegionsOnServer(ss);
        List<RegionInfo> tableRegions = new ArrayList<>();
        for (RegionInfo regionInfo : regions) {
            if (regionInfo.getTable().equals(tableName)) {
                tableRegions.add(regionInfo);
            }
        }
        return tableRegions;
    }, rsn, changed);
    t1.start();
    t1.join();
    t2.join();
    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {

        @Override
        public boolean evaluate() {
            if (changed.get()) {
                boolean serverHasTableRegions = false;
                for (RegionInfo regionInfo : master.getreplacedignmentManager().getRegionsOnServer(ss)) {
                    if (regionInfo.getTable().equals(tableName)) {
                        serverHasTableRegions = true;
                        break;
                    }
                }
                return !serverHasTableRegions && !rsn.getRegionLocation().equals(ss);
            }
            return false;
        }
    });
}

18 View Complete Implementation : DefaultVisibilityLabelServiceImpl.java
Copyright Apache License 2.0
Author : apache
private static void getSortedTagOrdinals(List<List<Integer>> fullTagsList, Tag tag) throws IOException {
    List<Integer> tagsOrdinalInSortedOrder = new ArrayList<>();
    int offset = tag.getValueOffset();
    int endOffset = offset + tag.getValueLength();
    while (offset < endOffset) {
        Pair<Integer, Integer> result = TagUtil.readVIntValuePart(tag, offset);
        tagsOrdinalInSortedOrder.add(result.getFirst());
        offset += result.getSecond();
    }
    Collections.sort(tagsOrdinalInSortedOrder);
    fullTagsList.add(tagsOrdinalInSortedOrder);
}

18 View Complete Implementation : DefaultVisibilityLabelServiceImpl.java
Copyright Apache License 2.0
Author : apache
@Override
public void init(RegionCoprocessorEnvironment e) throws IOException {
    /* So, presumption that the RegionCE has a ZK Connection is too much. Why would a RCE have
     * a ZK instance? This is cheating presuming we have access to the RS ZKW. TODO: Fix.
     *
     * And what is going on here? This ain't even a Coprocessor? And its being preplaceded a CP Env?
     */
    // This is a CoreCoprocessor. On creation, we should have gotten an environment that
    // implements HasRegionServerServices so we can get at RSS. FIX!!!! Integrate this CP as
    // native service.
    ZKWatcher zk = ((HasRegionServerServices) e).getRegionServerServices().getZooKeeper();
    try {
        labelsCache = VisibilityLabelsCache.createAndGet(zk, this.conf);
    } catch (IOException ioe) {
        LOG.error("Error creating VisibilityLabelsCache", ioe);
        throw ioe;
    }
    this.scanLabelGenerators = VisibilityUtils.getScanLabelGenerators(this.conf);
    if (e.getRegion().getRegionInfo().getTable().equals(LABELS_TABLE_NAME)) {
        this.labelsRegion = e.getRegion();
        Pair<Map<String, Integer>, Map<String, List<Integer>>> labelsAndUserAuths = extractLabelsAndAuths(getExistingLabelsWithAuths());
        Map<String, Integer> labels = labelsAndUserAuths.getFirst();
        Map<String, List<Integer>> userAuths = labelsAndUserAuths.getSecond();
        // Add the "system" label if it is not added into the system yet
        addSystemLabel(this.labelsRegion, labels, userAuths);
        // Ordinal 1 is reserved for "system" label.
        int ordinal = SYSTEM_LABEL_ORDINAL;
        for (Integer i : labels.values()) {
            if (i > ordinal) {
                ordinal = i;
            }
        }
        this.ordinalCounter.set(ordinal + 1);
        if (labels.size() > 0) {
            // If there is no data need not write to zk
            byte[] serialized = VisibilityUtils.getDataToWriteToZooKeeper(labels);
            this.labelsCache.writeToZookeeper(serialized, true);
            this.labelsCache.refreshLabelsCache(serialized);
        }
        if (userAuths.size() > 0) {
            byte[] serialized = VisibilityUtils.getUserAuthsDataToWriteToZooKeeper(userAuths);
            this.labelsCache.writeToZookeeper(serialized, false);
            this.labelsCache.refreshUserAuthsCache(serialized);
        }
    }
}

18 View Complete Implementation : DefaultVisibilityLabelServiceImpl.java
Copyright Apache License 2.0
Author : apache
protected void updateZk(boolean labelAddition) throws IOException {
    // We will add to zookeeper here.
    // TODO we should add the delta only to zk. Else this will be a very heavy op and when there are
    // so many labels and auth in the system, we will end up adding lots of data to zk. Most
    // possibly we will exceed zk node data limit!
    Pair<Map<String, Integer>, Map<String, List<Integer>>> labelsAndUserAuths = extractLabelsAndAuths(getExistingLabelsWithAuths());
    Map<String, Integer> existingLabels = labelsAndUserAuths.getFirst();
    Map<String, List<Integer>> userAuths = labelsAndUserAuths.getSecond();
    if (labelAddition) {
        byte[] serialized = VisibilityUtils.getDataToWriteToZooKeeper(existingLabels);
        this.labelsCache.writeToZookeeper(serialized, true);
    } else {
        byte[] serialized = VisibilityUtils.getUserAuthsDataToWriteToZooKeeper(userAuths);
        this.labelsCache.writeToZookeeper(serialized, false);
    }
}

18 View Complete Implementation : TestFuzzyRowAndColumnRangeFilter.java
Copyright Apache License 2.0
Author : apache
private void runTest(Table hTable, int cqStart, int expectedSize) throws IOException {
    // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
    byte[] fuzzyKey = new byte[10];
    ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
    buf.clear();
    buf.putShort((short) 2);
    for (int i = 0; i < 4; i++) buf.put((byte) 63);
    buf.putInt((short) 1);
    byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
    Pair<byte[], byte[]> pair = new Pair<>(fuzzyKey, mask);
    FuzzyRowFilter fuzzyRowFilter = new FuzzyRowFilter(Lists.newArrayList(pair));
    ColumnRangeFilter columnRangeFilter = new ColumnRangeFilter(Bytes.toBytes(cqStart), true, Bytes.toBytes(4), true);
    // regular test
    runScanner(hTable, expectedSize, fuzzyRowFilter, columnRangeFilter);
    // reverse filter order test
    runScanner(hTable, expectedSize, columnRangeFilter, fuzzyRowFilter);
}

18 View Complete Implementation : BulkLoadHFilesTool.java
Copyright Apache License 2.0
Author : apache
private Map<LoadQueueItem, ByteBuffer> performBulkLoad(AsyncClusterConnection conn, TableName tableName, Deque<LoadQueueItem> queue, ExecutorService pool, boolean copyFile) throws IOException {
    int count = 0;
    fsDelegationToken.acquireDelegationToken(queue.peek().getFilePath().getFileSystem(getConf()));
    bulkToken = FutureUtils.get(conn.prepareBulkLoad(tableName));
    Pair<Multimap<ByteBuffer, LoadQueueItem>, Set<String>> pair = null;
    Map<LoadQueueItem, ByteBuffer> item2RegionMap = new HashMap<>();
    // replacedumes that region splits can happen while this occurs.
    while (!queue.isEmpty()) {
        // need to reload split keys each iteration.
        final List<Pair<byte[], byte[]>> startEndKeys = FutureUtils.get(conn.getRegionLocator(tableName).getStartEndKeys());
        if (count != 0) {
            LOG.info("Split occurred while grouping HFiles, retry attempt " + count + " with " + queue.size() + " files remaining to group or split");
        }
        int maxRetries = getConf().getInt(HConstants.BULKLOAD_MAX_RETRIES_NUMBER, 10);
        maxRetries = Math.max(maxRetries, startEndKeys.size() + 1);
        if (maxRetries != 0 && count >= maxRetries) {
            throw new IOException("Retry attempted " + count + " times without completing, bailing out");
        }
        count++;
        // Using ByteBuffer for byte[] equality semantics
        pair = groupOrSplitPhase(conn, tableName, pool, queue, startEndKeys);
        Multimap<ByteBuffer, LoadQueueItem> regionGroups = pair.getFirst();
        if (!checkHFilesCountPerRegionPerFamily(regionGroups)) {
            // Error is logged inside checkHFilesCountPerRegionPerFamily.
            throw new IOException("Trying to load more than " + maxFilesPerRegionPerFamily + " hfiles to one family of one region");
        }
        bulkLoadPhase(conn, tableName, queue, regionGroups, copyFile, item2RegionMap);
    // NOTE: The next iteration's split / group could happen in parallel to
    // atomic bulkloads replaceduming that there are splits and no merges, and
    // that we can atomically pull out the groups we want to retry.
    }
    return item2RegionMap;
}

18 View Complete Implementation : VisibilityController.java
Copyright Apache License 2.0
Author : apache
@Override
public List<Pair<Cell, Cell>> postIncrementBeforeWAL(ObserverContext<RegionCoprocessorEnvironment> ctx, Mutation mutation, List<Pair<Cell, Cell>> cellPairs) throws IOException {
    List<Pair<Cell, Cell>> resultPairs = new ArrayList<>(cellPairs.size());
    for (Pair<Cell, Cell> pair : cellPairs) {
        resultPairs.add(new Pair<>(pair.getFirst(), createNewCellWithTags(mutation, pair.getSecond())));
    }
    return resultPairs;
}

17 View Complete Implementation : SyncReplicationWALProvider.java
Copyright Apache License 2.0
Author : apache
@Override
public void init(WALFactory factory, Configuration conf, String providerId) throws IOException {
    if (!initialized.compareAndSet(false, true)) {
        throw new IllegalStateException("WALProvider.init should only be called once.");
    }
    provider.init(factory, conf, providerId);
    this.conf = conf;
    this.factory = factory;
    Pair<EventLoopGroup, Clreplaced<? extends Channel>> eventLoopGroupAndChannelClreplaced = NettyAsyncFSWALConfigHelper.getEventLoopConfig(conf);
    eventLoopGroup = eventLoopGroupAndChannelClreplaced.getFirst();
    channelClreplaced = eventLoopGroupAndChannelClreplaced.getSecond();
}

17 View Complete Implementation : TestRegionMergeTransactionOnCluster.java
Copyright Apache License 2.0
Author : apache
@Test
public void testMergeWithReplicas() throws Exception {
    final TableName tableName = TableName.valueOf(name.getMethodName());
    // Create table and load data.
    createTableAndLoadData(MASTER, tableName, 5, 2);
    List<Pair<RegionInfo, ServerName>> initialRegionToServers = MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getConnection(), tableName);
    // Merge 1st and 2nd region
    PairOfSameType<RegionInfo> mergedRegions = mergeRegionsAndVerifyRegionNum(MASTER, tableName, 0, 2, 5 * 2 - 2);
    List<Pair<RegionInfo, ServerName>> currentRegionToServers = MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getConnection(), tableName);
    List<RegionInfo> initialRegions = new ArrayList<>();
    for (Pair<RegionInfo, ServerName> p : initialRegionToServers) {
        initialRegions.add(p.getFirst());
    }
    List<RegionInfo> currentRegions = new ArrayList<>();
    for (Pair<RegionInfo, ServerName> p : currentRegionToServers) {
        currentRegions.add(p.getFirst());
    }
    // this is the first region
    replacedertTrue(initialRegions.contains(mergedRegions.getFirst()));
    replacedertTrue(initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(mergedRegions.getFirst(), // this is the replica of the first region
    1)));
    // this is the second region
    replacedertTrue(initialRegions.contains(mergedRegions.getSecond()));
    replacedertTrue(initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(mergedRegions.getSecond(), // this is the replica of the second region
    1)));
    // this is the new region
    replacedertTrue(!initialRegions.contains(currentRegions.get(0)));
    replacedertTrue(!initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(currentRegions.get(0), // replica of the new region
    1)));
    replacedertTrue(currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(currentRegions.get(0), // replica of the new region
    1)));
    replacedertTrue(!currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(mergedRegions.getFirst(), // replica of the merged region
    1)));
    replacedertTrue(!currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(mergedRegions.getSecond(), // replica of the merged region
    1)));
}

17 View Complete Implementation : TestExportSnapshotHelpers.java
Copyright Apache License 2.0
Author : apache
private void verifyBalanceSplit(final List<Pair<SnapshotFileInfo, Long>> split, final String[] expected, final long expectedSize) {
    replacedertEquals(expected.length, split.size());
    long totalSize = 0;
    for (int i = 0; i < expected.length; ++i) {
        Pair<SnapshotFileInfo, Long> fileInfo = split.get(i);
        replacedertEquals(expected[i], fileInfo.getFirst().getHfile());
        totalSize += fileInfo.getSecond();
    }
    replacedertEquals(expectedSize, totalSize);
}

17 View Complete Implementation : PermissionStorage.java
Copyright Apache License 2.0
Author : apache
/**
 * Loads all of the permission grants stored in a region of the {@code _acl_}
 * table.
 *
 * @param aclRegion the acl region
 * @return a map of the permissions for this table.
 * @throws IOException if an error occurs
 */
static Map<byte[], ListMultimap<String, UserPermission>> loadAll(Region aclRegion) throws IOException {
    if (!isAclRegion(aclRegion)) {
        throw new IOException("Can only load permissions from " + ACL_TABLE_NAME);
    }
    Map<byte[], ListMultimap<String, UserPermission>> allPerms = new TreeMap<>(Bytes.BYTES_RAWCOMPARATOR);
    // do a full scan of _acl_ table
    Scan scan = new Scan();
    scan.addFamily(ACL_LIST_FAMILY);
    InternalScanner iScanner = null;
    try {
        iScanner = aclRegion.getScanner(scan);
        while (true) {
            List<Cell> row = new ArrayList<>();
            boolean hasNext = iScanner.next(row);
            ListMultimap<String, UserPermission> perms = ArrayListMultimap.create();
            byte[] entry = null;
            for (Cell kv : row) {
                if (entry == null) {
                    entry = CellUtil.cloneRow(kv);
                }
                Pair<String, Permission> permissionsOfUserOnTable = parsePermissionRecord(entry, kv, null, null, false, null);
                if (permissionsOfUserOnTable != null) {
                    String username = permissionsOfUserOnTable.getFirst();
                    Permission permission = permissionsOfUserOnTable.getSecond();
                    perms.put(username, new UserPermission(username, permission));
                }
            }
            if (entry != null) {
                allPerms.put(entry, perms);
            }
            if (!hasNext) {
                break;
            }
        }
    } finally {
        if (iScanner != null) {
            iScanner.close();
        }
    }
    return allPerms;
}

17 View Complete Implementation : AbstractTestRegionLocator.java
Copyright Apache License 2.0
Author : apache
@Test
public void testStartEndKeys() throws IOException {
    replacedertStartKeys(getStartKeys(TABLE_NAME));
    replacedertEndKeys(getEndKeys(TABLE_NAME));
    Pair<byte[][], byte[][]> startEndKeys = getStartEndKeys(TABLE_NAME);
    replacedertStartKeys(startEndKeys.getFirst());
    replacedertEndKeys(startEndKeys.getSecond());
}

17 View Complete Implementation : DefaultVisibilityLabelServiceImpl.java
Copyright Apache License 2.0
Author : apache
/**
 * @param tags
 *          - all the visibility tags replacedociated with the current Cell
 * @return - the modified visibility expression as byte[]
 */
private byte[] createModifiedVisExpression(final List<Tag> tags) throws IOException {
    StringBuilder visibilityString = new StringBuilder();
    for (Tag tag : tags) {
        if (tag.getType() == TagType.VISIBILITY_TAG_TYPE) {
            if (visibilityString.length() != 0) {
                visibilityString.append(VisibilityConstants.CLOSED_PARAN).append(VisibilityConstants.OR_OPERATOR);
            }
            int offset = tag.getValueOffset();
            int endOffset = offset + tag.getValueLength();
            boolean expressionStart = true;
            while (offset < endOffset) {
                Pair<Integer, Integer> result = TagUtil.readVIntValuePart(tag, offset);
                int currLabelOrdinal = result.getFirst();
                if (currLabelOrdinal < 0) {
                    int temp = -currLabelOrdinal;
                    String label = this.labelsCache.getLabel(temp);
                    if (expressionStart) {
                        // Quote every label in case of unicode characters if present
                        visibilityString.append(VisibilityConstants.OPEN_PARAN).append(VisibilityConstants.NOT_OPERATOR).append(CellVisibility.quote(label));
                    } else {
                        visibilityString.append(VisibilityConstants.AND_OPERATOR).append(VisibilityConstants.NOT_OPERATOR).append(CellVisibility.quote(label));
                    }
                } else {
                    String label = this.labelsCache.getLabel(currLabelOrdinal);
                    if (expressionStart) {
                        visibilityString.append(VisibilityConstants.OPEN_PARAN).append(CellVisibility.quote(label));
                    } else {
                        visibilityString.append(VisibilityConstants.AND_OPERATOR).append(CellVisibility.quote(label));
                    }
                }
                expressionStart = false;
                offset += result.getSecond();
            }
        }
    }
    if (visibilityString.length() != 0) {
        visibilityString.append(VisibilityConstants.CLOSED_PARAN);
        // Return the string formed as byte[]
        return Bytes.toBytes(visibilityString.toString());
    }
    return null;
}

17 View Complete Implementation : AggregationClient.java
Copyright Apache License 2.0
Author : apache
/**
 * This is the client side interface/handle for calling the average method for
 * a given cf-cq combination. It was necessary to add one more call stack as
 * its return type should be a decimal value, irrespective of what
 * columninterpreter says. So, this methods collects the necessary parameters
 * to compute the average and returs the double value.
 * @param table table to scan.
 * @param ci the user's ColumnInterpreter implementation
 * @param scan the HBase scan object to use to read data from HBase
 * @return <R, S>
 * @throws Throwable The caller is supposed to handle the exception as they are thrown
 *           & propagated to it.
 */
public <R, S, P extends Message, Q extends Message, T extends Message> double avg(final Table table, final ColumnInterpreter<R, S, P, Q, T> ci, Scan scan) throws Throwable {
    Pair<S, Long> p = getAvgArgs(table, ci, scan);
    return ci.divideForAvg(p.getFirst(), p.getSecond());
}

17 View Complete Implementation : TestFuzzyRowFilterEndToEnd.java
Copyright Apache License 2.0
Author : apache
private void runTest1(Table hTable) throws IOException {
    // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
    byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
    List<Pair<byte[], byte[]>> list = new ArrayList<>();
    for (int i = 0; i < totalFuzzyKeys; i++) {
        byte[] fuzzyKey = new byte[10];
        ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
        buf.clear();
        buf.putShort((short) 2);
        for (int j = 0; j < 4; j++) {
            buf.put(fuzzyValue);
        }
        buf.putInt(i);
        Pair<byte[], byte[]> pair = new Pair<>(fuzzyKey, mask);
        list.add(pair);
    }
    int expectedSize = secondPartCardinality * totalFuzzyKeys * colQualifiersTotal;
    FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
    // Filters are not stateless - we can't reuse them
    FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
    // regular test
    runScanner(hTable, expectedSize, fuzzyRowFilter0);
    // optimized from block cache
    runScanner(hTable, expectedSize, fuzzyRowFilter1);
}

17 View Complete Implementation : AggregationClient.java
Copyright Apache License 2.0
Author : apache
/**
 * This is the client side interface/handle for calling the average method for
 * a given cf-cq combination. It was necessary to add one more call stack as
 * its return type should be a decimal value, irrespective of what
 * columninterpreter says. So, this methods collects the necessary parameters
 * to compute the average and returs the double value.
 * @param tableName the name of the table to scan
 * @param ci the user's ColumnInterpreter implementation
 * @param scan the HBase scan object to use to read data from HBase
 * @return <R, S>
 * @throws Throwable The caller is supposed to handle the exception as they are thrown
 *           & propagated to it.
 */
public <R, S, P extends Message, Q extends Message, T extends Message> double avg(final TableName tableName, final ColumnInterpreter<R, S, P, Q, T> ci, Scan scan) throws Throwable {
    Pair<S, Long> p = getAvgArgs(tableName, ci, scan);
    return ci.divideForAvg(p.getFirst(), p.getSecond());
}

17 View Complete Implementation : ExportSnapshot.java
Copyright Apache License 2.0
Author : apache
/**
 * Given a list of file paths and sizes, create around ngroups in as balanced a way as possible.
 * The groups created will have similar amounts of bytes.
 * <p>
 * The algorithm used is pretty straightforward; the file list is sorted by size,
 * and then each group fetch the bigger file available, iterating through groups
 * alternating the direction.
 */
static List<List<Pair<SnapshotFileInfo, Long>>> getBalancedSplits(final List<Pair<SnapshotFileInfo, Long>> files, final int ngroups) {
    // Sort files by size, from small to big
    Collections.sort(files, new Comparator<Pair<SnapshotFileInfo, Long>>() {

        public int compare(Pair<SnapshotFileInfo, Long> a, Pair<SnapshotFileInfo, Long> b) {
            long r = a.getSecond() - b.getSecond();
            return (r < 0) ? -1 : ((r > 0) ? 1 : 0);
        }
    });
    // create balanced groups
    List<List<Pair<SnapshotFileInfo, Long>>> fileGroups = new LinkedList<>();
    long[] sizeGroups = new long[ngroups];
    int hi = files.size() - 1;
    int lo = 0;
    List<Pair<SnapshotFileInfo, Long>> group;
    int dir = 1;
    int g = 0;
    while (hi >= lo) {
        if (g == fileGroups.size()) {
            group = new LinkedList<>();
            fileGroups.add(group);
        } else {
            group = fileGroups.get(g);
        }
        Pair<SnapshotFileInfo, Long> fileInfo = files.get(hi--);
        // add the hi one
        sizeGroups[g] += fileInfo.getSecond();
        group.add(fileInfo);
        // change direction when at the end or the beginning
        g += dir;
        if (g == ngroups) {
            dir = -1;
            g = ngroups - 1;
        } else if (g < 0) {
            dir = 1;
            g = 0;
        }
    }
    if (LOG.isDebugEnabled()) {
        for (int i = 0; i < sizeGroups.length; ++i) {
            LOG.debug("export split=" + i + " size=" + StringUtils.humanReadableInt(sizeGroups[i]));
        }
    }
    return fileGroups;
}

17 View Complete Implementation : SyncReplicationWALProvider.java
Copyright Apache License 2.0
Author : apache
@Override
public WAL getWAL(RegionInfo region) throws IOException {
    if (region == null) {
        return provider.getWAL(null);
    }
    WAL wal = null;
    Optional<Pair<String, String>> peerIdAndRemoteWALDir = peerInfoProvider.getPeerIdAndRemoteWALDir(region.getTable());
    if (peerIdAndRemoteWALDir.isPresent()) {
        Pair<String, String> pair = peerIdAndRemoteWALDir.get();
        wal = getWAL(pair.getFirst(), pair.getSecond());
    }
    return wal != null ? wal : provider.getWAL(region);
}

17 View Complete Implementation : BulkLoadHFilesTool.java
Copyright Apache License 2.0
Author : apache
/**
 * Attempt to replacedign the given load queue item into its target region group. If the hfile boundary
 * no longer fits into a region, physically splits the hfile such that the new bottom half will
 * fit and returns the list of LQI's corresponding to the resultant hfiles.
 * <p/>
 * protected for testing
 * @throws IOException if an IO failure is encountered
 */
@VisibleForTesting
protected Pair<List<LoadQueueItem>, String> groupOrSplit(AsyncClusterConnection conn, TableName tableName, Multimap<ByteBuffer, LoadQueueItem> regionGroups, LoadQueueItem item, List<Pair<byte[], byte[]>> startEndKeys) throws IOException {
    Path hfilePath = item.getFilePath();
    Optional<byte[]> first, last;
    try (HFile.Reader hfr = HFile.createReader(hfilePath.getFileSystem(getConf()), hfilePath, CacheConfig.DISABLED, true, getConf())) {
        first = hfr.getFirstRowKey();
        last = hfr.getLastRowKey();
    } catch (FileNotFoundException fnfe) {
        LOG.debug("encountered", fnfe);
        return new Pair<>(null, hfilePath.getName());
    }
    LOG.info("Trying to load hfile=" + hfilePath + " first=" + first.map(Bytes::toStringBinary) + " last=" + last.map(Bytes::toStringBinary));
    if (!first.isPresent() || !last.isPresent()) {
        replacedert !first.isPresent() && !last.isPresent();
        // TODO what if this is due to a bad HFile?
        LOG.info("hfile " + hfilePath + " has no entries, skipping");
        return null;
    }
    if (Bytes.compareTo(first.get(), last.get()) > 0) {
        throw new IllegalArgumentException("Invalid range: " + Bytes.toStringBinary(first.get()) + " > " + Bytes.toStringBinary(last.get()));
    }
    int idx = Collections.binarySearch(startEndKeys, Pair.newPair(first.get(), HConstants.EMPTY_END_ROW), (p1, p2) -> Bytes.compareTo(p1.getFirst(), p2.getFirst()));
    if (idx < 0) {
        // not on boundary, returns -(insertion index). Calculate region it
        // would be in.
        idx = -(idx + 1) - 1;
    }
    int indexForCallable = idx;
    /*
     * we can consider there is a region hole in following conditions. 1) if idx < 0,then first
     * region info is lost. 2) if the endkey of a region is not equal to the startkey of the next
     * region. 3) if the endkey of the last region is not empty.
     */
    if (indexForCallable < 0) {
        throw new IOException("The first region info for table " + tableName + " can't be found in hbase:meta.Please use hbck tool to fix it first.");
    } else if ((indexForCallable == startEndKeys.size() - 1) && !Bytes.equals(startEndKeys.get(indexForCallable).getSecond(), HConstants.EMPTY_BYTE_ARRAY)) {
        throw new IOException("The last region info for table " + tableName + " can't be found in hbase:meta.Please use hbck tool to fix it first.");
    } else if (indexForCallable + 1 < startEndKeys.size() && !(Bytes.compareTo(startEndKeys.get(indexForCallable).getSecond(), startEndKeys.get(indexForCallable + 1).getFirst()) == 0)) {
        throw new IOException("The endkey of one region for table " + tableName + " is not equal to the startkey of the next region in hbase:meta." + "Please use hbck tool to fix it first.");
    }
    boolean lastKeyInRange = Bytes.compareTo(last.get(), startEndKeys.get(idx).getSecond()) < 0 || Bytes.equals(startEndKeys.get(idx).getSecond(), HConstants.EMPTY_BYTE_ARRAY);
    if (!lastKeyInRange) {
        Pair<byte[], byte[]> startEndKey = startEndKeys.get(indexForCallable);
        List<LoadQueueItem> lqis = splitStoreFile(item, FutureUtils.get(conn.getAdmin().getDescriptor(tableName)), startEndKey.getSecond());
        return new Pair<>(lqis, null);
    }
    // group regions.
    regionGroups.put(ByteBuffer.wrap(startEndKeys.get(idx).getFirst()), item);
    return null;
}

17 View Complete Implementation : TestSecureRESTServer.java
Copyright Apache License 2.0
Author : apache
@Test
public void testNegativeAuthorization() throws Exception {
    Pair<CloseableHttpClient, HttpClientContext> pair = getClient();
    CloseableHttpClient client = pair.getFirst();
    HttpClientContext context = pair.getSecond();
    StringEnreplacedy enreplacedy = new StringEnreplacedy("{\"name\":\"test\", \"ColumnSchema\":[{\"name\":\"f\"}]}", ContentType.APPLICATION_JSON);
    HttpPut put = new HttpPut("http://localhost:" + REST_TEST.getServletPort() + "/test/schema");
    put.setEnreplacedy(enreplacedy);
    UserGroupInformation unprivileged = UserGroupInformation.loginUserFromKeytabAndReturnUGI(CLIENT_PRINCIPAL, clientKeytab.getAbsolutePath());
    unprivileged.doAs(new PrivilegedExceptionAction<Void>() {

        @Override
        public Void run() throws Exception {
            try (CloseableHttpResponse response = client.execute(put, context)) {
                final int statusCode = response.getStatusLine().getStatusCode();
                HttpEnreplacedy enreplacedy = response.getEnreplacedy();
                replacedertEquals("Got response: " + EnreplacedyUtils.toString(enreplacedy), HttpURLConnection.HTTP_FORBIDDEN, statusCode);
            }
            return null;
        }
    });
}

17 View Complete Implementation : BulkLoadHFilesTool.java
Copyright Apache License 2.0
Author : apache
/**
 * @param conn the HBase cluster connection
 * @param tableName the table name of the table to load into
 * @param pool the ExecutorService
 * @param queue the queue for LoadQueueItem
 * @param startEndKeys start and end keys
 * @return A map that groups LQI by likely bulk load region targets and Set of missing hfiles.
 */
private Pair<Multimap<ByteBuffer, LoadQueueItem>, Set<String>> groupOrSplitPhase(AsyncClusterConnection conn, TableName tableName, ExecutorService pool, Deque<LoadQueueItem> queue, List<Pair<byte[], byte[]>> startEndKeys) throws IOException {
    // <region start key, LQI> need synchronized only within this scope of this
    // phase because of the puts that happen in futures.
    Multimap<ByteBuffer, LoadQueueItem> rgs = HashMultimap.create();
    final Multimap<ByteBuffer, LoadQueueItem> regionGroups = Multimaps.synchronizedMultimap(rgs);
    Set<String> missingHFiles = new HashSet<>();
    Pair<Multimap<ByteBuffer, LoadQueueItem>, Set<String>> pair = new Pair<>(regionGroups, missingHFiles);
    // drain LQIs and figure out bulk load groups
    Set<Future<Pair<List<LoadQueueItem>, String>>> splittingFutures = new HashSet<>();
    while (!queue.isEmpty()) {
        final LoadQueueItem item = queue.remove();
        final Callable<Pair<List<LoadQueueItem>, String>> call = () -> groupOrSplit(conn, tableName, regionGroups, item, startEndKeys);
        splittingFutures.add(pool.submit(call));
    }
    // get all the results. All grouping and splitting must finish before
    // we can attempt the atomic loads.
    for (Future<Pair<List<LoadQueueItem>, String>> lqis : splittingFutures) {
        try {
            Pair<List<LoadQueueItem>, String> splits = lqis.get();
            if (splits != null) {
                if (splits.getFirst() != null) {
                    queue.addAll(splits.getFirst());
                } else {
                    missingHFiles.add(splits.getSecond());
                }
            }
        } catch (ExecutionException e1) {
            Throwable t = e1.getCause();
            if (t instanceof IOException) {
                LOG.error("IOException during splitting", e1);
                // would have been thrown if not parallelized,
                throw (IOException) t;
            }
            LOG.error("Unexpected execution exception during splitting", e1);
            throw new IllegalStateException(t);
        } catch (InterruptedException e1) {
            LOG.error("Unexpected interrupted exception during splitting", e1);
            throw (InterruptedIOException) new InterruptedIOException().initCause(e1);
        }
    }
    return pair;
}

17 View Complete Implementation : TestFuzzyRowFilterEndToEnd.java
Copyright Apache License 2.0
Author : apache
private void runTest2(Table hTable) throws IOException {
    // [0, 0, ?, ?, ?, ?, 0, 0, 0, 0] , [0, 1, ?, ?, ?, ?, 0, 0, 0, 1]...
    byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
    List<Pair<byte[], byte[]>> list = new ArrayList<>();
    for (int i = 0; i < totalFuzzyKeys; i++) {
        byte[] fuzzyKey = new byte[10];
        ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
        buf.clear();
        buf.putShort((short) (i * 2));
        for (int j = 0; j < 4; j++) {
            buf.put(fuzzyValue);
        }
        buf.putInt(i * 2);
        Pair<byte[], byte[]> pair = new Pair<>(fuzzyKey, mask);
        list.add(pair);
    }
    int expectedSize = totalFuzzyKeys * secondPartCardinality * colQualifiersTotal;
    FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
    // Filters are not stateless - we can't reuse them
    FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
    // regular test
    runScanner(hTable, expectedSize, fuzzyRowFilter0);
    // optimized from block cache
    runScanner(hTable, expectedSize, fuzzyRowFilter1);
}

17 View Complete Implementation : PeriodicRandomActionPolicy.java
Copyright Apache License 2.0
Author : apache
@Override
public void init(PolicyContext context) throws Exception {
    super.init(context);
    for (Pair<Action, Integer> action : actions) {
        action.getFirst().init(this.context);
    }
}

17 View Complete Implementation : DefaultVisibilityLabelServiceImpl.java
Copyright Apache License 2.0
Author : apache
@Override
public List<String> listLabels(String regex) throws IOException {
    replacedert (labelsRegion != null);
    Pair<Map<String, Integer>, Map<String, List<Integer>>> labelsAndUserAuths = extractLabelsAndAuths(getExistingLabelsWithAuths());
    Map<String, Integer> labels = labelsAndUserAuths.getFirst();
    labels.remove(SYSTEM_LABEL);
    if (regex != null) {
        Pattern pattern = Pattern.compile(regex);
        ArrayList<String> matchedLabels = new ArrayList<>();
        for (String label : labels.keySet()) {
            if (pattern.matcher(label).matches()) {
                matchedLabels.add(label);
            }
        }
        return matchedLabels;
    }
    return new ArrayList<>(labels.keySet());
}

17 View Complete Implementation : SnapshotScannerHDFSAclController.java
Copyright Apache License 2.0
Author : apache
private void removeUserGlobalHdfsAcl(Table aclTable, String userName, UserPermission userPermission) throws IOException {
    if (SnapshotScannerHDFSAclStorage.hasUserGlobalHdfsAcl(aclTable, userName)) {
        // 1. Get namespaces and tables which global user acls are already synced
        Pair<Set<String>, Set<TableName>> namespaceAndTable = SnapshotScannerHDFSAclStorage.getUserNamespaceAndTable(aclTable, userName);
        Set<String> skipNamespaces = namespaceAndTable.getFirst();
        Set<TableName> skipTables = namespaceAndTable.getSecond().stream().filter(t -> !skipNamespaces.contains(t.getNamespacereplacedtring())).collect(Collectors.toSet());
        // 2. Remove user HDFS acls(skip namespaces and tables directories
        // whose acl must be reversed)
        hdfsAclHelper.revokeAcl(userPermission, skipNamespaces, skipTables);
        // 3. Remove global user acl is synced to HDFS in acl table
        SnapshotScannerHDFSAclStorage.deleteUserGlobalHdfsAcl(aclTable, userName);
    }
}

17 View Complete Implementation : TestZKProcedure.java
Copyright Apache License 2.0
Author : apache
private void closeAll(ProcedureCoordinator coordinator, ZKProcedureCoordinator coordinatorController, List<Pair<ProcedureMember, ZKProcedureMemberRpcs>> cohort) throws IOException {
    // make sure we close all the resources
    for (Pair<ProcedureMember, ZKProcedureMemberRpcs> member : cohort) {
        member.getFirst().close();
        member.getSecond().close();
    }
    coordinator.close();
    coordinatorController.close();
}

17 View Complete Implementation : AsyncFSWALProvider.java
Copyright Apache License 2.0
Author : apache
@Override
protected void doInit(Configuration conf) throws IOException {
    Pair<EventLoopGroup, Clreplaced<? extends Channel>> eventLoopGroupAndChannelClreplaced = NettyAsyncFSWALConfigHelper.getEventLoopConfig(conf);
    eventLoopGroup = eventLoopGroupAndChannelClreplaced.getFirst();
    channelClreplaced = eventLoopGroupAndChannelClreplaced.getSecond();
}

16 View Complete Implementation : Constraints.java
Copyright Apache License 2.0
Author : apache
/**
 * Add constraints and their replacedociated configurations to the table.
 * <p>
 * Adding the same constraint clreplaced twice will overwrite the first
 * constraint's configuration
 * <p>
 * Each constraint, when added to the table, will have a specific priority,
 * dictating the order in which the {@link Constraint} will be run. A
 * {@link Constraint} earlier in the list will be run before those later in
 * the list. The same logic applies between two Constraints over time (earlier
 * added is run first on the regionserver).
 *
 * @param desc
 *          {@link HTableDescriptor} to add a {@link Constraint}
 * @param constraints
 *          {@link Pair} of a {@link Constraint} and its replacedociated
 *          {@link Configuration}. The Constraint will be configured on load
 *          with the specified configuration.All constraints are considered
 *          automatically enabled on add
 * @throws IOException
 *           if any constraint could not be deserialized. replacedumes if 1
 *           constraint is not loaded properly, something has gone terribly
 *           wrong and that all constraints need to be enforced.
 */
public static void add(HTableDescriptor desc, Pair<Clreplaced<? extends Constraint>, Configuration>... constraints) throws IOException {
    enable(desc);
    long priority = getNextPriority(desc);
    for (Pair<Clreplaced<? extends Constraint>, Configuration> pair : constraints) {
        addConstraint(desc, pair.getFirst(), pair.getSecond(), priority++);
    }
    updateLatestPriority(desc, priority);
}