Here are the examples of the java api org.apache.hadoop.util.ReflectionUtils.newInstance() taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
155 Examples
19
View Complete Implementation : EntityGroupFSTimelineStore.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
private TimelineStore createSummaryStore() {
return ReflectionUtils.newInstance(getConfig().getClreplaced(YarnConfiguration.TIMELINE_SERVICE_ENreplacedYGROUP_FS_STORE_SUMMARY_STORE, LeveldbTimelineStore.clreplaced, TimelineStore.clreplaced), getConfig());
}
19
View Complete Implementation : SCMStoreBaseTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void TestZeroArgConstructor() throws Exception {
// Test that the SCMStore implementation clreplaced is compatible with
// ReflectionUtils#newInstance
ReflectionUtils.newInstance(getStoreClreplaced(), new Configuration());
}
19
View Complete Implementation : CsiAdaptorFactory.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Load csi-driver-adaptor from configuration. If the configuration is not
* specified, the default implementation
* for the adaptor is {@link DefaultCsiAdaptorImpl}. If the configured clreplaced
* is not a valid variation of {@link CsiAdaptorPlugin} or the clreplaced cannot
* be found, this function will throw a RuntimeException.
* @param driverName
* @param conf
* @return CsiAdaptorPlugin
* @throws YarnException if unable to create the adaptor clreplaced.
* @throws RuntimeException if given clreplaced is not found or not
* an instance of {@link CsiAdaptorPlugin}
*/
public static CsiAdaptorPlugin getAdaptor(String driverName, Configuration conf) throws YarnException {
// load configuration
String configName = YarnConfiguration.NM_CSI_ADAPTOR_PREFIX + driverName + YarnConfiguration.NM_CSI_ADAPTOR_CLreplaced;
Clreplaced<? extends CsiAdaptorPlugin> impl = conf.getClreplaced(configName, DefaultCsiAdaptorImpl.clreplaced, CsiAdaptorPlugin.clreplaced);
if (impl == null) {
throw new YarnException("Unable to init csi-adaptor from the" + " clreplaced specified via " + configName);
}
// init the adaptor
CsiAdaptorPlugin instance = ReflectionUtils.newInstance(impl, conf);
LOG.info("csi-adaptor initiated, implementation: " + impl.getCanonicalName());
return instance;
}
19
View Complete Implementation : AuxiliaryServiceWithCustomClassLoader.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static AuxiliaryServiceWithCustomClreplacedLoader getInstance(Configuration conf, String clreplacedName, String appClreplacedPath, String[] systemClreplacedes) throws IOException, ClreplacedNotFoundException {
ClreplacedLoader customClreplacedLoader = createAuxServiceClreplacedLoader(appClreplacedPath, systemClreplacedes);
Clreplaced<?> clazz = Clreplaced.forName(clreplacedName, true, customClreplacedLoader);
Clreplaced<? extends AuxiliaryService> sClreplaced = clazz.replacedubclreplaced(AuxiliaryService.clreplaced);
AuxiliaryService wrapped = ReflectionUtils.newInstance(sClreplaced, conf);
return new AuxiliaryServiceWithCustomClreplacedLoader(clreplacedName + " with custom clreplaced loader", wrapped, customClreplacedLoader);
}
19
View Complete Implementation : TestAvailableSpaceVolumeChoosingPolicy.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test(timeout = 60000)
public void testFourUnbalancedVolumes() throws Exception {
@SuppressWarnings("unchecked")
final AvailableSpaceVolumeChoosingPolicy<FsVolumeSpi> policy = ReflectionUtils.newInstance(AvailableSpaceVolumeChoosingPolicy.clreplaced, null);
List<FsVolumeSpi> volumes = new ArrayList<FsVolumeSpi>();
// First volume with 1MB free space
volumes.add(Mockito.mock(FsVolumeSpi.clreplaced));
Mockito.when(volumes.get(0).getAvailable()).thenReturn(1024L * 1024L);
// Second volume with 1MB + 1 byte free space
volumes.add(Mockito.mock(FsVolumeSpi.clreplaced));
Mockito.when(volumes.get(1).getAvailable()).thenReturn(1024L * 1024L + 1);
// Third volume with 3MB free space, which is a difference of 2MB, more
// than the threshold of 1MB.
volumes.add(Mockito.mock(FsVolumeSpi.clreplaced));
Mockito.when(volumes.get(2).getAvailable()).thenReturn(1024L * 1024L * 3);
// Fourth volume, again with 3MB free space.
volumes.add(Mockito.mock(FsVolumeSpi.clreplaced));
Mockito.when(volumes.get(3).getAvailable()).thenReturn(1024L * 1024L * 3);
// We should alternate replacedigning between the two volumes with a lot of free
// space.
initPolicy(policy, 1.0f);
replacedert.replacedertEquals(volumes.get(2), policy.chooseVolume(volumes, 100, null));
replacedert.replacedertEquals(volumes.get(3), policy.chooseVolume(volumes, 100, null));
replacedert.replacedertEquals(volumes.get(2), policy.chooseVolume(volumes, 100, null));
replacedert.replacedertEquals(volumes.get(3), policy.chooseVolume(volumes, 100, null));
// We should alternate replacedigning between the two volumes with less free
// space.
initPolicy(policy, 0.0f);
replacedert.replacedertEquals(volumes.get(0), policy.chooseVolume(volumes, 100, null));
replacedert.replacedertEquals(volumes.get(1), policy.chooseVolume(volumes, 100, null));
replacedert.replacedertEquals(volumes.get(0), policy.chooseVolume(volumes, 100, null));
replacedert.replacedertEquals(volumes.get(1), policy.chooseVolume(volumes, 100, null));
}
19
View Complete Implementation : AppAdminClient.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* <p>
* Create a new instance of AppAdminClient.
* </p>
*
* @param appType application type
* @param conf configuration
* @return app admin client
*/
@Public
@Unstable
public static AppAdminClient createAppAdminClient(String appType, Configuration conf) {
Map<String, String> clientClreplacedMap = conf.getPropsWithPrefix(YARN_APP_ADMIN_CLIENT_PREFIX);
if (!clientClreplacedMap.containsKey(DEFAULT_TYPE)) {
clientClreplacedMap.put(DEFAULT_TYPE, DEFAULT_CLreplaced_NAME);
}
if (!clientClreplacedMap.containsKey(UNIT_TEST_TYPE)) {
clientClreplacedMap.put(UNIT_TEST_TYPE, UNIT_TEST_CLreplaced_NAME);
}
if (!clientClreplacedMap.containsKey(appType)) {
throw new IllegalArgumentException("App admin client clreplaced name not " + "specified for type " + appType);
}
String clientClreplacedName = clientClreplacedMap.get(appType);
Clreplaced<? extends AppAdminClient> clientClreplaced;
try {
clientClreplaced = (Clreplaced<? extends AppAdminClient>) Clreplaced.forName(clientClreplacedName);
} catch (ClreplacedNotFoundException e) {
throw new YarnRuntimeException("Invalid app admin client clreplaced", e);
}
AppAdminClient appAdminClient = ReflectionUtils.newInstance(clientClreplaced, conf);
appAdminClient.init(conf);
appAdminClient.start();
return appAdminClient;
}
19
View Complete Implementation : TimelineV1DelegationTokenSecretManagerService.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
protected TimelineStateStore createStateStore(Configuration conf) {
return ReflectionUtils.newInstance(conf.getClreplaced(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLreplaced, LeveldbTimelineStateStore.clreplaced, TimelineStateStore.clreplaced), conf);
}
19
View Complete Implementation : ResourceCalculatorPlugin.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Create the ResourceCalculatorPlugin from the clreplaced name and configure it. If
* clreplaced name is null, this method will try and return a memory calculator
* plugin available for this system.
*
* @param clazz ResourceCalculator plugin clreplaced-name
* @param conf configure the plugin with this.
* @return ResourceCalculatorPlugin or null if ResourceCalculatorPlugin is not
* available for current system
*/
public static ResourceCalculatorPlugin getResourceCalculatorPlugin(Clreplaced<? extends ResourceCalculatorPlugin> clazz, Configuration conf) {
if (clazz != null) {
return ReflectionUtils.newInstance(clazz, conf);
}
try {
return new ResourceCalculatorPlugin();
} catch (UnsupportedOperationException ue) {
LOG.warn("Failed to instantiate default resource calculator. " + ue.getMessage());
} catch (Throwable t) {
LOG.warn(t + ": Failed to instantiate default resource calculator.", t);
}
return null;
}
19
View Complete Implementation : MapFile.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This method attempts to fix a corrupt MapFile by re-creating its index.
* @param fs filesystem
* @param dir directory containing the MapFile data and index
* @param keyClreplaced key clreplaced (has to be a subclreplaced of Writable)
* @param valueClreplaced value clreplaced (has to be a subclreplaced of Writable)
* @param dryrun do not perform any changes, just report what needs to be done
* @return number of valid entries in this MapFile, or -1 if no fixing was needed
* @throws Exception
*/
public static long fix(FileSystem fs, Path dir, Clreplaced<? extends Writable> keyClreplaced, Clreplaced<? extends Writable> valueClreplaced, boolean dryrun, Configuration conf) throws Exception {
String dr = (dryrun ? "[DRY RUN ] " : "");
Path data = new Path(dir, DATA_FILE_NAME);
Path index = new Path(dir, INDEX_FILE_NAME);
int indexInterval = conf.getInt(Writer.INDEX_INTERVAL, 128);
if (!fs.exists(data)) {
// there's nothing we can do to fix this!
throw new Exception(dr + "Missing data file in " + dir + ", impossible to fix this.");
}
if (fs.exists(index)) {
// no fixing needed
return -1;
}
SequenceFile.Reader dataReader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(data));
if (!dataReader.getKeyClreplaced().equals(keyClreplaced)) {
throw new Exception(dr + "Wrong key clreplaced in " + dir + ", expected" + keyClreplaced.getName() + ", got " + dataReader.getKeyClreplaced().getName());
}
if (!dataReader.getValueClreplaced().equals(valueClreplaced)) {
throw new Exception(dr + "Wrong value clreplaced in " + dir + ", expected" + valueClreplaced.getName() + ", got " + dataReader.getValueClreplaced().getName());
}
long cnt = 0L;
Writable key = ReflectionUtils.newInstance(keyClreplaced, conf);
Writable value = ReflectionUtils.newInstance(valueClreplaced, conf);
SequenceFile.Writer indexWriter = null;
if (!dryrun) {
indexWriter = SequenceFile.createWriter(conf, SequenceFile.Writer.file(index), SequenceFile.Writer.keyClreplaced(keyClreplaced), SequenceFile.Writer.valueClreplaced(LongWritable.clreplaced));
}
try {
/**
* What's the position (in bytes) we wrote when we got the last index
*/
long lastIndexPos = -1;
/**
* What was size when we last wrote an index. Set to MIN_VALUE to ensure
* that we have an index at position zero - midKey will throw an exception
* if this is not the case
*/
long lastIndexKeyCount = Long.MIN_VALUE;
long pos = dataReader.getPosition();
LongWritable position = new LongWritable();
long nextBlock = pos;
boolean blockCompressed = dataReader.isBlockCompressed();
while (dataReader.next(key, value)) {
if (blockCompressed) {
long curPos = dataReader.getPosition();
if (curPos > nextBlock) {
// current block position
pos = nextBlock;
nextBlock = curPos;
}
}
// Follow the same logic as in
// {@link MapFile.Writer#append(WritableComparable, Writable)}
if (cnt >= lastIndexKeyCount + indexInterval && pos > lastIndexPos) {
position.set(pos);
if (!dryrun) {
indexWriter.append(key, position);
}
lastIndexPos = pos;
lastIndexKeyCount = cnt;
}
if (!blockCompressed) {
// next record position
pos = dataReader.getPosition();
}
cnt++;
}
} catch (Throwable t) {
// truncated data file. swallow it.
}
dataReader.close();
if (!dryrun)
indexWriter.close();
return cnt;
}
19
View Complete Implementation : TrashPolicy.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Get an instance of the configured TrashPolicy based on the value
* of the configuration parameter fs.trash.clreplacedname.
*
* @param conf the configuration to be used
* @param fs the file system to be used
* @return an instance of TrashPolicy
*/
public static TrashPolicy getInstance(Configuration conf, FileSystem fs) {
Clreplaced<? extends TrashPolicy> trashClreplaced = conf.getClreplaced("fs.trash.clreplacedname", TrashPolicyDefault.clreplaced, TrashPolicy.clreplaced);
TrashPolicy trash = ReflectionUtils.newInstance(trashClreplaced, conf);
// initialize TrashPolicy
trash.initialize(conf, fs);
return trash;
}
19
View Complete Implementation : TestAvailableSpaceVolumeChoosingPolicy.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
// Test the Round-Robin block-volume fallback path when all volumes are within
// the threshold.
@Test(timeout = 60000)
public void testRR() throws Exception {
@SuppressWarnings("unchecked")
final AvailableSpaceVolumeChoosingPolicy<FsVolumeSpi> policy = ReflectionUtils.newInstance(AvailableSpaceVolumeChoosingPolicy.clreplaced, null);
initPolicy(policy, 1.0f);
TestRoundRobinVolumeChoosingPolicy.testRR(policy);
}
19
View Complete Implementation : TestAvailableSpaceVolumeChoosingPolicy.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test(timeout = 60000)
public void testThreeUnbalancedVolumes() throws Exception {
@SuppressWarnings("unchecked")
final AvailableSpaceVolumeChoosingPolicy<FsVolumeSpi> policy = ReflectionUtils.newInstance(AvailableSpaceVolumeChoosingPolicy.clreplaced, null);
List<FsVolumeSpi> volumes = new ArrayList<FsVolumeSpi>();
// First volume with 1MB free space
volumes.add(Mockito.mock(FsVolumeSpi.clreplaced));
Mockito.when(volumes.get(0).getAvailable()).thenReturn(1024L * 1024L);
// Second volume with 3MB free space, which is a difference of 2MB, more
// than the threshold of 1MB.
volumes.add(Mockito.mock(FsVolumeSpi.clreplaced));
Mockito.when(volumes.get(1).getAvailable()).thenReturn(1024L * 1024L * 3);
// Third volume, again with 3MB free space.
volumes.add(Mockito.mock(FsVolumeSpi.clreplaced));
Mockito.when(volumes.get(2).getAvailable()).thenReturn(1024L * 1024L * 3);
// We should alternate replacedigning between the two volumes with a lot of free
// space.
initPolicy(policy, 1.0f);
replacedert.replacedertEquals(volumes.get(1), policy.chooseVolume(volumes, 100, null));
replacedert.replacedertEquals(volumes.get(2), policy.chooseVolume(volumes, 100, null));
replacedert.replacedertEquals(volumes.get(1), policy.chooseVolume(volumes, 100, null));
replacedert.replacedertEquals(volumes.get(2), policy.chooseVolume(volumes, 100, null));
// All writes should be replacedigned to the volume with the least free space.
initPolicy(policy, 0.0f);
replacedert.replacedertEquals(volumes.get(0), policy.chooseVolume(volumes, 100, null));
replacedert.replacedertEquals(volumes.get(0), policy.chooseVolume(volumes, 100, null));
replacedert.replacedertEquals(volumes.get(0), policy.chooseVolume(volumes, 100, null));
replacedert.replacedertEquals(volumes.get(0), policy.chooseVolume(volumes, 100, null));
}
19
View Complete Implementation : TestBinaryPartitioner.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testLowerBound() {
Configuration conf = new Configuration();
BinaryParreplacedioner.setLeftOffset(conf, 0);
BinaryParreplacedioner<?> parreplacedioner = ReflectionUtils.newInstance(BinaryParreplacedioner.clreplaced, conf);
BinaryComparable key1 = new BytesWritable(new byte[] { 1, 2, 3, 4, 5 });
BinaryComparable key2 = new BytesWritable(new byte[] { 6, 2, 3, 4, 5 });
int parreplacedion1 = parreplacedioner.getParreplacedion(key1, null, 10);
int parreplacedion2 = parreplacedioner.getParreplacedion(key2, null, 10);
replacedertTrue(parreplacedion1 != parreplacedion2);
}
19
View Complete Implementation : TrustedChannelResolver.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Returns an instance of TrustedChannelResolver.
* Looks up the configuration to see if there is custom clreplaced specified.
* @return TrustedChannelResolver
*/
public static TrustedChannelResolver getInstance(Configuration conf) {
Clreplaced<? extends TrustedChannelResolver> clazz = conf.getClreplaced(HdfsClientConfigKeys.DFS_TRUSTEDCHANNEL_RESOLVER_CLreplaced, TrustedChannelResolver.clreplaced, TrustedChannelResolver.clreplaced);
return ReflectionUtils.newInstance(clazz, conf);
}
19
View Complete Implementation : YarnConfigurationStoreFactory.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static YarnConfigurationStore getStore(Configuration conf) {
String store = conf.get(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLreplaced, YarnConfiguration.MEMORY_CONFIGURATION_STORE);
switch(store) {
case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
return new InMemoryConfigurationStore();
case YarnConfiguration.LEVELDB_CONFIGURATION_STORE:
return new LeveldbConfigurationStore();
case YarnConfiguration.ZK_CONFIGURATION_STORE:
return new ZKConfigurationStore();
case YarnConfiguration.FS_CONFIGURATION_STORE:
return new FSSchedulerConfigurationStore();
default:
Clreplaced<? extends YarnConfigurationStore> storeClreplaced = conf.getClreplaced(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLreplaced, InMemoryConfigurationStore.clreplaced, YarnConfigurationStore.clreplaced);
LOG.info("Using YarnConfigurationStore implementation - " + storeClreplaced);
return ReflectionUtils.newInstance(storeClreplaced, conf);
}
}
19
View Complete Implementation : StateStoreSerializer.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
private static StateStoreSerializer newSerializer(final Configuration conf) {
Clreplaced<? extends StateStoreSerializer> serializerName = conf.getClreplaced(RBFConfigKeys.FEDERATION_STORE_SERIALIZER_CLreplaced, RBFConfigKeys.FEDERATION_STORE_SERIALIZER_CLreplaced_DEFAULT, StateStoreSerializer.clreplaced);
return ReflectionUtils.newInstance(serializerName, conf);
}
19
View Complete Implementation : TrashPolicy.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Get an instance of the configured TrashPolicy based on the value
* of the configuration parameter fs.trash.clreplacedname.
*
* @param conf the configuration to be used
* @param fs the file system to be used
* @param home the home directory
* @return an instance of TrashPolicy
* @deprecated Use {@link #getInstance(Configuration, FileSystem)} instead.
*/
@Deprecated
public static TrashPolicy getInstance(Configuration conf, FileSystem fs, Path home) {
Clreplaced<? extends TrashPolicy> trashClreplaced = conf.getClreplaced("fs.trash.clreplacedname", TrashPolicyDefault.clreplaced, TrashPolicy.clreplaced);
TrashPolicy trash = ReflectionUtils.newInstance(trashClreplaced, conf);
// initialize TrashPolicy
trash.initialize(conf, fs, home);
return trash;
}
19
View Complete Implementation : RMStateStoreFactory.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static RMStateStore getStore(Configuration conf) {
Clreplaced<? extends RMStateStore> storeClreplaced = conf.getClreplaced(YarnConfiguration.RM_STORE, MemoryRMStateStore.clreplaced, RMStateStore.clreplaced);
LOG.info("Using RMStateStore implementation - " + storeClreplaced);
return ReflectionUtils.newInstance(storeClreplaced, conf);
}
19
View Complete Implementation : PlacementFactory.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Create a new {@link PlacementRule} based on the rule clreplaced from the
* configuration. This is used to instantiate rules by the scheduler which
* does not resolve the clreplaced before this call.
* @param ruleStr The name of the clreplaced to instantiate
* @param conf The configuration object to set for the rule
* @return Created clreplaced instance
*/
public static PlacementRule getPlacementRule(String ruleStr, Configuration conf) throws ClreplacedNotFoundException {
Clreplaced<? extends PlacementRule> ruleClreplaced = Clreplaced.forName(ruleStr).replacedubclreplaced(PlacementRule.clreplaced);
LOG.info("Using PlacementRule implementation - " + ruleClreplaced);
return ReflectionUtils.newInstance(ruleClreplaced, conf);
}
19
View Complete Implementation : FileSystemImage.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Override
public int run(String[] argv) throws Exception {
Options options = options();
CommandLineParser parser = new PosixParser();
CommandLine cmd;
try {
cmd = parser.parse(options, argv);
} catch (ParseException e) {
System.out.println("Error parsing command-line options: " + e.getMessage());
printUsage();
return -1;
}
if (cmd.hasOption("h")) {
printUsage();
return -1;
}
ImageWriter.Options opts = ReflectionUtils.newInstance(ImageWriter.Options.clreplaced, getConf());
for (Option o : cmd.getOptions()) {
switch(o.getOpt()) {
case "o":
opts.output(o.getValue());
break;
case "u":
opts.ugi(Clreplaced.forName(o.getValue()).replacedubclreplaced(UGIResolver.clreplaced));
break;
case "b":
opts.blocks(Clreplaced.forName(o.getValue()).replacedubclreplaced(BlockAliasMap.clreplaced));
break;
case "i":
opts.blockIds(Clreplaced.forName(o.getValue()).replacedubclreplaced(BlockResolver.clreplaced));
break;
case "c":
opts.cache(Integer.parseInt(o.getValue()));
break;
case "cid":
opts.clusterID(o.getValue());
break;
case "bpid":
opts.blockPoolID(o.getValue());
break;
default:
throw new UnsupportedOperationException("Unknown option: " + o.getOpt());
}
}
String[] rem = cmd.getArgs();
if (rem.length != 1) {
printUsage();
return -1;
}
try (ImageWriter w = new ImageWriter(opts)) {
for (TreePath e : new FSTreeWalk(new Path(rem[0]), getConf())) {
// add and continue
w.accept(e);
}
}
return 0;
}
19
View Complete Implementation : ProxyUsers.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Returns an instance of ImpersonationProvider.
* Looks up the configuration to see if there is custom clreplaced specified.
* @param conf
* @return ImpersonationProvider
*/
private static ImpersonationProvider getInstance(Configuration conf) {
Clreplaced<? extends ImpersonationProvider> clazz = conf.getClreplaced(CommonConfigurationKeysPublic.HADOOP_SECURITY_IMPERSONATION_PROVIDER_CLreplaced, DefaultImpersonationProvider.clreplaced, ImpersonationProvider.clreplaced);
return ReflectionUtils.newInstance(clazz, conf);
}
19
View Complete Implementation : TestBinaryPartitioner.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testDefaultOffsets() {
Configuration conf = new Configuration();
BinaryParreplacedioner<?> parreplacedioner = ReflectionUtils.newInstance(BinaryParreplacedioner.clreplaced, conf);
BinaryComparable key1 = new BytesWritable(new byte[] { 1, 2, 3, 4, 5 });
BinaryComparable key2 = new BytesWritable(new byte[] { 1, 2, 3, 4, 5 });
int parreplacedion1 = parreplacedioner.getParreplacedion(key1, null, 10);
int parreplacedion2 = parreplacedioner.getParreplacedion(key2, null, 10);
replacedertEquals(parreplacedion1, parreplacedion2);
key1 = new BytesWritable(new byte[] { 1, 2, 3, 4, 5 });
key2 = new BytesWritable(new byte[] { 6, 2, 3, 4, 5 });
parreplacedion1 = parreplacedioner.getParreplacedion(key1, null, 10);
parreplacedion2 = parreplacedioner.getParreplacedion(key2, null, 10);
replacedertTrue(parreplacedion1 != parreplacedion2);
key1 = new BytesWritable(new byte[] { 1, 2, 3, 4, 5 });
key2 = new BytesWritable(new byte[] { 1, 2, 3, 4, 6 });
parreplacedion1 = parreplacedioner.getParreplacedion(key1, null, 10);
parreplacedion2 = parreplacedioner.getParreplacedion(key2, null, 10);
replacedertTrue(parreplacedion1 != parreplacedion2);
}
19
View Complete Implementation : RamDiskReplicaTracker.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Get an instance of the configured RamDiskReplicaTracker based on the
* the configuration property
* {@link org.apache.hadoop.hdfs.DFSConfigKeys#DFS_DATANODE_RAM_DISK_REPLICA_TRACKER_KEY}.
*
* @param conf the configuration to be used
* @param dataset the FsDataset object.
* @return an instance of RamDiskReplicaTracker
*/
static RamDiskReplicaTracker getInstance(final Configuration conf, final FsDatasetImpl fsDataset) {
final Clreplaced<? extends RamDiskReplicaTracker> trackerClreplaced = conf.getClreplaced(DFSConfigKeys.DFS_DATANODE_RAM_DISK_REPLICA_TRACKER_KEY, DFSConfigKeys.DFS_DATANODE_RAM_DISK_REPLICA_TRACKER_DEFAULT, RamDiskReplicaTracker.clreplaced);
final RamDiskReplicaTracker tracker = ReflectionUtils.newInstance(trackerClreplaced, conf);
tracker.initialize(fsDataset);
return tracker;
}
19
View Complete Implementation : WritableUtils.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Make a copy of a writable object using serialization to a buffer.
* @param orig The object to copy
* @return The copied object
*/
public static <T extends Writable> T clone(T orig, Configuration conf) {
try {
// Unchecked cast from Clreplaced to Clreplaced<T>
@SuppressWarnings("unchecked")
T newInst = ReflectionUtils.newInstance((Clreplaced<T>) orig.getClreplaced(), conf);
ReflectionUtils.copy(conf, orig, newInst);
return newInst;
} catch (IOException e) {
throw new RuntimeException("Error writing/reading clone buffer", e);
}
}
19
View Complete Implementation : CryptoCodec.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Get crypto codec for specified algorithm/mode/padding.
*
* @param conf
* the configuration
* @param cipherSuite
* algorithm/mode/padding
* @return CryptoCodec the codec object. Null value will be returned if no
* crypto codec clreplacedes with cipher suite configured.
*/
public static CryptoCodec getInstance(Configuration conf, CipherSuite cipherSuite) {
List<Clreplaced<? extends CryptoCodec>> klreplacedes = getCodecClreplacedes(conf, cipherSuite);
if (klreplacedes == null) {
return null;
}
CryptoCodec codec = null;
for (Clreplaced<? extends CryptoCodec> klreplaced : klreplacedes) {
try {
CryptoCodec c = ReflectionUtils.newInstance(klreplaced, conf);
if (c.getCipherSuite().getName().equals(cipherSuite.getName())) {
if (codec == null) {
PerformanceAdvisory.LOG.debug("Using crypto codec {}.", klreplaced.getName());
codec = c;
}
} else {
PerformanceAdvisory.LOG.debug("Crypto codec {} doesn't meet the cipher suite {}.", klreplaced.getName(), cipherSuite.getName());
}
} catch (Exception e) {
PerformanceAdvisory.LOG.debug("Crypto codec {} is not available.", klreplaced.getName());
}
}
return codec;
}
19
View Complete Implementation : TestRoundRobinVolumeChoosingPolicy.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
// Test the Round-Robin block-volume choosing algorithm.
@Test
public void testRR() throws Exception {
@SuppressWarnings("unchecked")
final RoundRobinVolumeChoosingPolicy<FsVolumeSpi> policy = ReflectionUtils.newInstance(RoundRobinVolumeChoosingPolicy.clreplaced, null);
testRR(policy);
}
19
View Complete Implementation : HistoryServerStateStoreServiceFactory.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Constructs an instance of the configured storage clreplaced
*
* @param conf the configuration
* @return the state storage instance
*/
public static HistoryServerStateStoreService getStore(Configuration conf) {
Clreplaced<? extends HistoryServerStateStoreService> storeClreplaced = HistoryServerNullStateStoreService.clreplaced;
boolean recoveryEnabled = conf.getBoolean(JHAdminConfig.MR_HS_RECOVERY_ENABLE, JHAdminConfig.DEFAULT_MR_HS_RECOVERY_ENABLE);
if (recoveryEnabled) {
storeClreplaced = conf.getClreplaced(JHAdminConfig.MR_HS_STATE_STORE, null, HistoryServerStateStoreService.clreplaced);
if (storeClreplaced == null) {
throw new RuntimeException("Unable to locate storage clreplaced, check " + JHAdminConfig.MR_HS_STATE_STORE);
}
}
return ReflectionUtils.newInstance(storeClreplaced, conf);
}
19
View Complete Implementation : SchedulingPolicy.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Returns a {@link SchedulingPolicy} instance corresponding
* to the preplaceded clazz.
*
* @param clazz a clreplaced that extends {@link SchedulingPolicy}
* @return a {@link SchedulingPolicy} instance
*/
public static SchedulingPolicy getInstance(Clreplaced<? extends SchedulingPolicy> clazz) {
SchedulingPolicy policy = ReflectionUtils.newInstance(clazz, null);
SchedulingPolicy policyRet = instances.putIfAbsent(clazz, policy);
if (policyRet != null) {
return policyRet;
}
return policy;
}
19
View Complete Implementation : ReduceTask.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
private CompressionCodec initCodec() {
// check if map-outputs are to be compressed
if (conf.getCompressMapOutput()) {
Clreplaced<? extends CompressionCodec> codecClreplaced = conf.getMapOutputCompressorClreplaced(DefaultCodec.clreplaced);
return ReflectionUtils.newInstance(codecClreplaced, conf);
}
return null;
}
19
View Complete Implementation : TestAvailableSpaceVolumeChoosingPolicy.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test(timeout = 60000)
public void testAvailableSpaceChanges() throws Exception {
@SuppressWarnings("unchecked")
final AvailableSpaceVolumeChoosingPolicy<FsVolumeSpi> policy = ReflectionUtils.newInstance(AvailableSpaceVolumeChoosingPolicy.clreplaced, null);
initPolicy(policy, 1.0f);
List<FsVolumeSpi> volumes = new ArrayList<FsVolumeSpi>();
// First volume with 1MB free space
volumes.add(Mockito.mock(FsVolumeSpi.clreplaced));
Mockito.when(volumes.get(0).getAvailable()).thenReturn(1024L * 1024L);
// Second volume with 3MB free space, which is a difference of 2MB, more
// than the threshold of 1MB.
volumes.add(Mockito.mock(FsVolumeSpi.clreplaced));
Mockito.when(volumes.get(1).getAvailable()).thenReturn(1024L * 1024L * 3).thenReturn(1024L * 1024L * 3).thenReturn(1024L * 1024L * 3).thenReturn(// After the third check, return 1MB.
1024L * 1024L * 1);
// Should still be able to get a volume for the replica even though the
// available space on the second volume changed.
replacedert.replacedertEquals(volumes.get(1), policy.chooseVolume(volumes, 100, null));
}
19
View Complete Implementation : DBRecordReader.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* @deprecated
*/
@Deprecated
public T createValue() {
return ReflectionUtils.newInstance(inputClreplaced, conf);
}
19
View Complete Implementation : DomainNameResolverFactory.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This function gets the instance based on the config.
*
* @param conf Configuration
* @param configKey config key name.
* @return Domain name resolver.
* @throws IOException when the clreplaced cannot be found or initiated.
*/
public static DomainNameResolver newInstance(Configuration conf, String configKey) {
Clreplaced<? extends DomainNameResolver> resolverClreplaced = conf.getClreplaced(configKey, DNSDomainNameResolver.clreplaced, DomainNameResolver.clreplaced);
return ReflectionUtils.newInstance(resolverClreplaced, conf);
}
19
View Complete Implementation : DelegatingLinuxContainerRuntime.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
private LinuxContainerRuntime createPluggableRuntime(Configuration conf, String runtimeType) throws ContainerExecutionException {
String confKey = String.format(YarnConfiguration.LINUX_CONTAINER_RUNTIME_CLreplaced_FMT, runtimeType);
Clreplaced<? extends LinuxContainerRuntime> clazz = conf.getClreplaced(confKey, null, LinuxContainerRuntime.clreplaced);
if (clazz == null) {
throw new ContainerExecutionException("Invalid runtime set in " + YarnConfiguration.LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES + " : " + runtimeType + " : Missing configuration " + confKey);
}
return ReflectionUtils.newInstance(clazz, conf);
}
19
View Complete Implementation : TestAvailableSpaceVolumeChoosingPolicy.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test(timeout = 60000)
public void testNotEnoughSpaceOnSelectedVolume() throws Exception {
@SuppressWarnings("unchecked")
final AvailableSpaceVolumeChoosingPolicy<FsVolumeSpi> policy = ReflectionUtils.newInstance(AvailableSpaceVolumeChoosingPolicy.clreplaced, null);
List<FsVolumeSpi> volumes = new ArrayList<FsVolumeSpi>();
// First volume with 1MB free space
volumes.add(Mockito.mock(FsVolumeSpi.clreplaced));
Mockito.when(volumes.get(0).getAvailable()).thenReturn(1024L * 1024L);
// Second volume with 3MB free space, which is a difference of 2MB, more
// than the threshold of 1MB.
volumes.add(Mockito.mock(FsVolumeSpi.clreplaced));
Mockito.when(volumes.get(1).getAvailable()).thenReturn(1024L * 1024L * 3);
// All writes should be replacedigned to the volume with the least free space.
// However, if the volume with the least free space doesn't have enough
// space to accept the replica size, and another volume does have enough
// free space, that should be chosen instead.
initPolicy(policy, 0.0f);
replacedert.replacedertEquals(volumes.get(1), policy.chooseVolume(volumes, 1024L * 1024L * 2, null));
}
19
View Complete Implementation : JobHistory.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
protected HistoryStorage createHistoryStorage() {
return ReflectionUtils.newInstance(conf.getClreplaced(JHAdminConfig.MR_HISTORY_STORAGE, CachedHistoryStorage.clreplaced, HistoryStorage.clreplaced), conf);
}
19
View Complete Implementation : TestShellBasedUnixGroupsMapping.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test(timeout = 4000)
public void testFiniteGroupResolutionTime() throws Exception {
Configuration conf = new Configuration();
String userName = "foobarnonexistinguser";
String commandTimeoutMessage = "ran longer than the configured timeout limit";
long testTimeout = 500L;
// Test a 1 second max-runtime timeout
conf.setLong(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY, testTimeout);
TestDelayedGroupCommand mapping = ReflectionUtils.newInstance(TestDelayedGroupCommand.clreplaced, conf);
ShellCommandExecutor executor = mapping.createGroupExecutor(userName);
replacedertEquals("Expected the group names executor to carry the configured timeout", testTimeout, executor.getTimeoutInterval());
executor = mapping.createGroupIDExecutor(userName);
replacedertEquals("Expected the group ID executor to carry the configured timeout", testTimeout, executor.getTimeoutInterval());
replacedertEquals("Expected no groups to be returned given a shell command timeout", 0, mapping.getGroups(userName).size());
replacedertTrue("Expected the logs to carry " + "a message about command timeout but was: " + shellMappingLog.getOutput(), shellMappingLog.getOutput().contains(commandTimeoutMessage));
shellMappingLog.clearOutput();
// Test also the parent Groups framework for expected behaviour
conf.setClreplaced(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, TestDelayedGroupCommand.clreplaced, GroupMappingServiceProvider.clreplaced);
Groups groups = new Groups(conf);
try {
groups.getGroups(userName);
fail("The groups framework call should " + "have failed with a command timeout");
} catch (IOException e) {
replacedertTrue("Expected the logs to carry " + "a message about command timeout but was: " + shellMappingLog.getOutput(), shellMappingLog.getOutput().contains(commandTimeoutMessage));
}
shellMappingLog.clearOutput();
// Test the no-timeout (default) configuration
conf = new Configuration();
long defaultTimeout = CommonConfigurationKeys.HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT;
mapping = ReflectionUtils.newInstance(TestDelayedGroupCommand.clreplaced, conf);
executor = mapping.createGroupExecutor(userName);
replacedertEquals("Expected the group names executor to carry the default timeout", defaultTimeout, executor.getTimeoutInterval());
executor = mapping.createGroupIDExecutor(userName);
replacedertEquals("Expected the group ID executor to carry the default timeout", defaultTimeout, executor.getTimeoutInterval());
mapping.getGroups(userName);
replacedertFalse("Didn't expect a timeout of command in execution but logs carry it: " + shellMappingLog.getOutput(), shellMappingLog.getOutput().contains(commandTimeoutMessage));
}
19
View Complete Implementation : WritableComparator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Construct a new {@link WritableComparable} instance.
*/
public WritableComparable newKey() {
return ReflectionUtils.newInstance(keyClreplaced, conf);
}
19
View Complete Implementation : Chain.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Configures all the chain elements for the task.
*
* @param jobConf chain job's JobConf.
*/
public void configure(JobConf jobConf) {
String prefix = getPrefix(isMap);
chainJobConf = jobConf;
SerializationFactory serializationFactory = new SerializationFactory(chainJobConf);
int index = jobConf.getInt(prefix + CHAIN_MAPPER_SIZE, 0);
for (int i = 0; i < index; i++) {
Clreplaced<? extends Mapper> klreplaced = jobConf.getClreplaced(prefix + CHAIN_MAPPER_CLreplaced + i, null, Mapper.clreplaced);
JobConf mConf = new JobConf(getChainElementConf(jobConf, prefix + CHAIN_MAPPER_CONFIG + i));
Mapper mapper = ReflectionUtils.newInstance(klreplaced, mConf);
mappers.add(mapper);
if (mConf.getBoolean(MAPPER_BY_VALUE, true)) {
mappersKeySerialization.add(serializationFactory.getSerialization(mConf.getClreplaced(MAPPER_OUTPUT_KEY_CLreplaced, null)));
mappersValueSerialization.add(serializationFactory.getSerialization(mConf.getClreplaced(MAPPER_OUTPUT_VALUE_CLreplaced, null)));
} else {
mappersKeySerialization.add(null);
mappersValueSerialization.add(null);
}
}
Clreplaced<? extends Reducer> klreplaced = jobConf.getClreplaced(prefix + CHAIN_REDUCER_CLreplaced, null, Reducer.clreplaced);
if (klreplaced != null) {
JobConf rConf = new JobConf(getChainElementConf(jobConf, prefix + CHAIN_REDUCER_CONFIG));
reducer = ReflectionUtils.newInstance(klreplaced, rConf);
if (rConf.getBoolean(REDUCER_BY_VALUE, true)) {
reducerKeySerialization = serializationFactory.getSerialization(rConf.getClreplaced(REDUCER_OUTPUT_KEY_CLreplaced, null));
reducerValueSerialization = serializationFactory.getSerialization(rConf.getClreplaced(REDUCER_OUTPUT_VALUE_CLreplaced, null));
} else {
reducerKeySerialization = null;
reducerValueSerialization = null;
}
}
}
19
View Complete Implementation : DFSNetworkTopology.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static DFSNetworkTopology getInstance(Configuration conf) {
DFSNetworkTopology nt = ReflectionUtils.newInstance(conf.getClreplaced(DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_KEY, DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_DEFAULT, DFSNetworkTopology.clreplaced), conf);
return (DFSNetworkTopology) nt.init(DFSTopologyNodeImpl.FACTORY);
}
19
View Complete Implementation : SaslPropertiesResolver.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Returns an instance of SaslPropertiesResolver.
* Looks up the configuration to see if there is custom clreplaced specified.
* Constructs the instance by preplaceding the configuration directly to the
* constructor to achieve thread safety using final fields.
* @param conf
* @return SaslPropertiesResolver
*/
public static SaslPropertiesResolver getInstance(Configuration conf) {
Clreplaced<? extends SaslPropertiesResolver> clazz = conf.getClreplaced(CommonConfigurationKeysPublic.HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLreplaced, SaslPropertiesResolver.clreplaced, SaslPropertiesResolver.clreplaced);
return ReflectionUtils.newInstance(clazz, conf);
}
19
View Complete Implementation : PipeMapRed.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
InputWriter createInputWriter(Clreplaced<? extends InputWriter> inputWriterClreplaced) throws IOException {
InputWriter inputWriter = ReflectionUtils.newInstance(inputWriterClreplaced, job_);
inputWriter.initialize(this);
return inputWriter;
}
19
View Complete Implementation : SignerManager.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Initialize custom signers and register them with the AWS SDK.
*/
public void initCustomSigners() {
String[] customSigners = ownerConf.getTrimmedStrings(CUSTOM_SIGNERS);
if (customSigners == null || customSigners.length == 0) {
// No custom signers specified, nothing to do.
LOG.debug("No custom signers specified");
return;
}
for (String customSigner : customSigners) {
String[] parts = customSigner.split(":");
if (!(parts.length == 1 || parts.length == 2 || parts.length == 3)) {
String message = "Invalid format (Expected name, name:SignerClreplaced," + " name:SignerClreplaced:SignerInitializerClreplaced)" + " for CustomSigner: [" + customSigner + "]";
LOG.error(message);
throw new IllegalArgumentException(message);
}
if (parts.length == 1) {
// Nothing to do. Trying to use a pre-defined Signer
} else {
// Register any custom Signer
maybeRegisterSigner(parts[0], parts[1], ownerConf);
// If an initializer is specified, take care of instantiating it and
// setting it up
if (parts.length == 3) {
Clreplaced<? extends AwsSignerInitializer> clazz = null;
try {
clazz = (Clreplaced<? extends AwsSignerInitializer>) ownerConf.getClreplacedByName(parts[2]);
} catch (ClreplacedNotFoundException e) {
throw new RuntimeException(String.format("SignerInitializer clreplaced" + " [%s] not found for signer [%s]", parts[2], parts[0]), e);
}
LOG.debug("Creating signer initializer: [{}] for signer: [{}]", parts[2], parts[0]);
AwsSignerInitializer signerInitializer = ReflectionUtils.newInstance(clazz, null);
initializers.add(signerInitializer);
signerInitializer.registerStore(bucketName, ownerConf, delegationTokenProvider, ownerUgi);
}
}
}
}
19
View Complete Implementation : NetworkTopology.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static NetworkTopology getInstance(Configuration conf, InnerNode.Factory factory) {
NetworkTopology nt = ReflectionUtils.newInstance(conf.getClreplaced(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY, NetworkTopology.clreplaced, NetworkTopology.clreplaced), conf);
return nt.init(factory);
}
19
View Complete Implementation : PlacementFactory.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Create a new {@link PlacementRule} based on the rule clreplaced from the
* configuration. This is used to instantiate rules by the scheduler which
* resolve the clreplaced before this call.
* @param ruleClreplaced The specific clreplaced reference to instantiate
* @param initArg The config to set
* @return Created clreplaced instance
*/
public static PlacementRule getPlacementRule(Clreplaced<? extends PlacementRule> ruleClreplaced, Object initArg) {
LOG.info("Creating PlacementRule implementation: " + ruleClreplaced);
PlacementRule rule = ReflectionUtils.newInstance(ruleClreplaced, null);
rule.setConfig(initArg);
return rule;
}
19
View Complete Implementation : ClientContext.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
private void initTopologyResolution(Configuration config) {
topologyResolutionEnabled = config.getBoolean(FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED, FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED_DEFAULT);
if (!topologyResolutionEnabled) {
return;
}
DNSToSwitchMapping dnsToSwitchMapping = ReflectionUtils.newInstance(config.getClreplaced(CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, ScriptBasedMapping.clreplaced, DNSToSwitchMapping.clreplaced), config);
String clientHostName = NetUtils.getLocalHostname();
List<String> nodes = new ArrayList<>();
nodes.add(clientHostName);
List<String> resolvedHosts = dnsToSwitchMapping.resolve(nodes);
if (resolvedHosts != null && !resolvedHosts.isEmpty() && !resolvedHosts.get(0).equals(NetworkTopology.DEFAULT_RACK)) {
// The client machine is able to resolve its own network location.
this.clientNode = new NodeBase(clientHostName, resolvedHosts.get(0));
}
}
19
View Complete Implementation : JobProperties.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Override
public Properties getAnonymizedValue(StatePool statePool, Configuration conf) {
Properties filteredProperties = null;
List<JobPropertyParser> pList = new ArrayList<JobPropertyParser>(1);
// load the parsers
String config = conf.get(PARSERS_CONFIG_KEY);
if (config != null) {
@SuppressWarnings("unchecked")
Clreplaced<JobPropertyParser>[] parsers = (Clreplaced[]) conf.getClreplacedes(PARSERS_CONFIG_KEY);
for (Clreplaced<JobPropertyParser> c : parsers) {
JobPropertyParser parser = ReflectionUtils.newInstance(c, conf);
pList.add(parser);
}
} else {
// add the default MapReduce filter
JobPropertyParser parser = new MapReduceJobPropertiesParser();
pList.add(parser);
}
// filter out the desired config key-value pairs
if (jobProperties != null) {
filteredProperties = new Properties();
// define a configuration object and load it with original job properties
for (Map.Entry<Object, Object> entry : jobProperties.entrySet()) {
// TODO Check for null key/value?
String key = entry.getKey().toString();
String value = entry.getValue().toString();
// find a parser for this key
for (JobPropertyParser p : pList) {
DataType<?> pValue = p.parseJobProperty(key, value);
if (pValue != null) {
filteredProperties.put(key, pValue);
break;
}
}
}
}
return filteredProperties;
}
19
View Complete Implementation : TestBinaryPartitioner.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testCustomOffsets() {
Configuration conf = new Configuration();
BinaryComparable key1 = new BytesWritable(new byte[] { 1, 2, 3, 4, 5 });
BinaryComparable key2 = new BytesWritable(new byte[] { 6, 2, 3, 7, 8 });
BinaryParreplacedioner.setOffsets(conf, 1, -3);
BinaryParreplacedioner<?> parreplacedioner = ReflectionUtils.newInstance(BinaryParreplacedioner.clreplaced, conf);
int parreplacedion1 = parreplacedioner.getParreplacedion(key1, null, 10);
int parreplacedion2 = parreplacedioner.getParreplacedion(key2, null, 10);
replacedertEquals(parreplacedion1, parreplacedion2);
BinaryParreplacedioner.setOffsets(conf, 1, 2);
parreplacedioner = ReflectionUtils.newInstance(BinaryParreplacedioner.clreplaced, conf);
parreplacedion1 = parreplacedioner.getParreplacedion(key1, null, 10);
parreplacedion2 = parreplacedioner.getParreplacedion(key2, null, 10);
replacedertEquals(parreplacedion1, parreplacedion2);
BinaryParreplacedioner.setOffsets(conf, -4, -3);
parreplacedioner = ReflectionUtils.newInstance(BinaryParreplacedioner.clreplaced, conf);
parreplacedion1 = parreplacedioner.getParreplacedion(key1, null, 10);
parreplacedion2 = parreplacedioner.getParreplacedion(key2, null, 10);
replacedertEquals(parreplacedion1, parreplacedion2);
}
19
View Complete Implementation : WritableFactories.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Create a new instance of a clreplaced with a defined factory.
*/
public static Writable newInstance(Clreplaced<? extends Writable> c, Configuration conf) {
WritableFactory factory = WritableFactories.getFactory(c);
if (factory != null) {
Writable result = factory.newInstance();
if (result instanceof Configurable) {
((Configurable) result).setConf(conf);
}
return result;
} else {
return ReflectionUtils.newInstance(c, conf);
}
}
19
View Complete Implementation : TestBinaryPartitioner.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testUpperBound() {
Configuration conf = new Configuration();
BinaryParreplacedioner.setRightOffset(conf, 4);
BinaryParreplacedioner<?> parreplacedioner = ReflectionUtils.newInstance(BinaryParreplacedioner.clreplaced, conf);
BinaryComparable key1 = new BytesWritable(new byte[] { 1, 2, 3, 4, 5 });
BinaryComparable key2 = new BytesWritable(new byte[] { 1, 2, 3, 4, 6 });
int parreplacedion1 = parreplacedioner.getParreplacedion(key1, null, 10);
int parreplacedion2 = parreplacedioner.getParreplacedion(key2, null, 10);
replacedertTrue(parreplacedion1 != parreplacedion2);
}
19
View Complete Implementation : TestAvailableSpaceVolumeChoosingPolicy.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test(timeout = 60000)
public void testTwoUnbalancedVolumes() throws Exception {
@SuppressWarnings("unchecked")
final AvailableSpaceVolumeChoosingPolicy<FsVolumeSpi> policy = ReflectionUtils.newInstance(AvailableSpaceVolumeChoosingPolicy.clreplaced, null);
initPolicy(policy, 1.0f);
List<FsVolumeSpi> volumes = new ArrayList<FsVolumeSpi>();
// First volume with 1MB free space
volumes.add(Mockito.mock(FsVolumeSpi.clreplaced));
Mockito.when(volumes.get(0).getAvailable()).thenReturn(1024L * 1024L);
// Second volume with 3MB free space, which is a difference of 2MB, more
// than the threshold of 1MB.
volumes.add(Mockito.mock(FsVolumeSpi.clreplaced));
Mockito.when(volumes.get(1).getAvailable()).thenReturn(1024L * 1024L * 3);
replacedert.replacedertEquals(volumes.get(1), policy.chooseVolume(volumes, 100, null));
replacedert.replacedertEquals(volumes.get(1), policy.chooseVolume(volumes, 100, null));
replacedert.replacedertEquals(volumes.get(1), policy.chooseVolume(volumes, 100, null));
}