Here are the examples of the java api org.apache.hadoop.mapreduce.TaskAttemptContext.getConfiguration() taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
155 Examples
19
View Complete Implementation : MagicS3GuardCommitter.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Did this task write any files in the work directory?
* Probes for a task existing by looking to see if the attempt dir exists.
* This adds more HTTP requests to the call. It may be better just to
* return true and rely on the commit task doing the work.
* @param context the task's context
* @return true if the attempt path exists
* @throws IOException failure to list the path
*/
@Override
public boolean needsTaskCommit(TaskAttemptContext context) throws IOException {
Path taskAttemptPath = getTaskAttemptPath(context);
try (DurationInfo d = new DurationInfo(LOG, "needsTaskCommit task %s", context.getTaskAttemptID())) {
return taskAttemptPath.getFileSystem(context.getConfiguration()).exists(taskAttemptPath);
}
}
19
View Complete Implementation : WikipediaDataTypeHandler.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Override
public void setup(TaskAttemptContext context) {
super.setup(context);
this.ingestHelper = (WikipediaIngestHelper) this.getHelper(null);
this.helper = this.ingestHelper.getDataTypeHelper();
try {
this.parser = factory.newDoreplacedentBuilder();
} catch (ParserConfigurationException e) {
throw new RuntimeException("Error instantiating DoreplacedentBuilder", e);
}
Configuration conf = context.getConfiguration();
this.counters = new ContentIndexCounters();
disableDCol = conf.getBoolean(OPT_NO_D_COL, disableDCol);
docWriterService = Executors.newSingleThreadExecutor();
try {
AcreplaceduloHelper acreplaceduloHelper = new AcreplaceduloHelper();
acreplaceduloHelper.setup(conf);
log.debug("Attempting to create Acreplacedulo connection.");
docWriter = acreplaceduloHelper.getConnector().createBatchWriter(conf.get("shard.table.name"), new BatchWriterConfig().setMaxLatency(60, TimeUnit.SECONDS).setMaxMemory(100000000L).setMaxWriteThreads(10));
log.debug("Created connection to Acreplacedulo for asynchronous doreplacedent storage.");
} catch (Exception e) {
log.warn("No doreplacedent payloads will be written to Acreplacedulo.", e);
// giving a stub batchwriter means I don't have a bunch of "if(writer != null)"s lying around
docWriter = new BatchWriter() {
@Override
public void addMutation(Mutation m) {
}
@Override
public void addMutations(Iterable<Mutation> iterable) throws MutationsRejectedException {
}
@Override
public void flush() throws MutationsRejectedException {
}
@Override
public void close() throws MutationsRejectedException {
}
};
}
}
19
View Complete Implementation : DelimitedTextInputFormat.java
Copyright Apache License 2.0
Author : marklogic
Copyright Apache License 2.0
Author : marklogic
@Override
public RecordReader<DoreplacedentURIWithSourceInfo, Text> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
if (isSplitInput(context.getConfiguration())) {
return new SplitDelimitedTextReader<Text>();
} else {
return new DelimitedTextReader<Text>();
}
}
19
View Complete Implementation : ContentOutputFormat.java
Copyright Apache License 2.0
Author : marklogic
Copyright Apache License 2.0
Author : marklogic
@Override
public RecordWriter<DoreplacedentURI, VALUEOUT> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
// TODO: if MAPREDUCE-3377 still exists, need to re-run initialize
fastLoad = Boolean.valueOf(conf.get(OUTPUT_FAST_LOAD));
Map<String, ContentSource> sourceMap = getSourceMap(fastLoad, context);
// construct the ContentWriter
return new ContentWriter<VALUEOUT>(conf, sourceMap, fastLoad, am);
}
19
View Complete Implementation : StagingCommitter.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Get the work path for a task.
* @param context job/task complex
* @param uuid UUID
* @return a path
* @throws IOException failure to build the path
*/
private static Path taskAttemptWorkingPath(TaskAttemptContext context, String uuid) throws IOException {
return getTaskAttemptPath(context, Paths.getLocalTaskAttemptTempDir(context.getConfiguration(), uuid, context.getTaskAttemptID()));
}
19
View Complete Implementation : GeoWaveDataTypeHandler.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Override
public void setup(final TaskAttemptContext context) {
final TypeRegistry registry = TypeRegistry.getInstance(context.getConfiguration());
final Type type = registry.get(context.getConfiguration().get(DataTypeHelperImpl.Properties.DATA_NAME));
conf = context.getConfiguration();
final String sftName = conf.get(type.typeName() + GEOWAVE_FEATURE_TYPE_NAME);
final String sftSpec = conf.get(type.typeName() + GEOWAVE_FEATURE_TYPE_SPEC);
try {
originalType = (sftSpec != null) ? DataUtilities.createType(sftName, sftSpec) : buildSimpleFeatureType(type.typeName());
} catch (SchemaException e) {
log.error("Unable to parse Simple Feature Type from spec [" + sftSpec + "]");
}
// create a simple feature data adapter
dataAdapter = new FeatureDataAdapter(originalType);
// determine whether features need to be reprojected
if (!GeoWaveGTDataStore.DEFAULT_CRS.equals(originalType.getCoordinateReferenceSystem())) {
reprojectedType = SimpleFeatureTypeBuilder.retype(originalType, GeoWaveGTDataStore.DEFAULT_CRS);
if (originalType.getCoordinateReferenceSystem() != null) {
try {
transform = CRS.findMathTransform(originalType.getCoordinateReferenceSystem(), GeoWaveGTDataStore.DEFAULT_CRS, true);
} catch (final FactoryException e) {
log.warn("Unable to create coordinate reference system transform", e);
}
}
}
// features are built using the original projection
builder = new SimpleFeatureBuilder(originalType);
// create an index
index = initIndex(conf.get(type.typeName() + GEOWAVE_INDEX_TYPE, null));
// namespace
tableNamespace = conf.get(type.typeName() + GEOWAVE_NAMESPACE, null);
// flatten geometry
flattenGeometry = conf.getBoolean(type.typeName() + GEOWAVE_FLATTEN_GEOMETRY, flattenGeometry);
// GeoWave metadata
metadata = new GeoWaveMetadata(tableNamespace, index, dataAdapter, originalType, reprojectedType, transform, flattenGeometry);
}
19
View Complete Implementation : AggregateXMLReader.java
Copyright Apache License 2.0
Author : marklogic
Copyright Apache License 2.0
Author : marklogic
protected void initAggConf(TaskAttemptContext context) {
Configuration conf = context.getConfiguration();
idName = conf.get(ConfigConstants.CONF_INPUT_URI_ID);
if (idName == null) {
useAutomaticId = true;
}
recordName = conf.get(ConfigConstants.CONF_AGGREGATE_RECORD_ELEMENT);
recordNamespace = conf.get(ConfigConstants.CONF_AGGREGATE_RECORD_NAMESPACE);
}
19
View Complete Implementation : MagicS3GuardCommitter.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Abort a task. Attempt load then abort all pending files,
* then try to delete the task attempt path.
* This method may be called on the job committer, rather than the
* task one (such as in the MapReduce AM after a task container failure).
* It must extract all paths and state from the preplaceded in context.
* @param context task context
* @throws IOException if there was some problem querying the path other
* than it not actually existing.
*/
@Override
public void abortTask(TaskAttemptContext context) throws IOException {
Path attemptPath = getTaskAttemptPath(context);
try (DurationInfo d = new DurationInfo(LOG, "Abort task %s", context.getTaskAttemptID())) {
getCommitOperations().abortAllSinglePendingCommits(attemptPath, true);
} finally {
deleteQuietly(attemptPath.getFileSystem(context.getConfiguration()), attemptPath, true);
destroyThreadPool();
}
}
19
View Complete Implementation : AtomDataTypeHandler.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Override
public void setup(TaskAttemptContext context) {
conf = context.getConfiguration();
tableName = ConfigurationHelper.isNull(context.getConfiguration(), ATOM_TABLE_NAME, String.clreplaced);
categoryTableName = tableName + "Categories";
subCategories = new HashMap<>();
markingFunctions = MarkingFunctions.Factory.createMarkingFunctions();
TypeRegistry.getInstance(context.getConfiguration());
String[] types = ConfigurationHelper.isNull(context.getConfiguration(), ATOM_TYPES_TO_PROCESS, String[].clreplaced);
// Set up the ingest helpers for the known datatypes.
fieldNames = ConfigurationHelper.isNull(context.getConfiguration(), ATOM_FIELD_NAMES, String[].clreplaced);
// Configuration.getStrings() eats empty values, we don't want to do that. Split it ourselves.
String aliases = ConfigurationHelper.isNull(context.getConfiguration(), ATOM_FIELD_ALIASES, String.clreplaced);
// keeps empty elements
fieldAliases = StringUtils.split(aliases, ',', true);
String overrides = ConfigurationHelper.isNull(context.getConfiguration(), ATOM_FIELD_VALUE_OVERRIDES, String.clreplaced);
// keeps empty elements
fieldOverrides = StringUtils.split(overrides, ',', true);
sCategories = StringUtils.split(ConfigurationHelper.isNull(context.getConfiguration(), ATOM_CATEGORY_SUB_FIELD, String.clreplaced), ',', false);
Set<String> tSet;
for (String s : sCategories) {
String[] field_value = StringUtils.split(s, ':', false);
if (field_value.length == 2 && (!Strings.isNullOrEmpty(field_value[0]) && !Strings.isNullOrEmpty(field_value[1]))) {
if (!subCategories.containsKey(field_value[0])) {
tSet = new HashSet<>();
} else {
tSet = subCategories.get(field_value[0]);
}
System.err.println("Value: " + field_value[0] + " " + field_value[1]);
tSet.add(field_value[1]);
subCategories.put(field_value[0], tSet);
}
}
// Make sure these 3 arrays are all the same size.
if (fieldNames.length != fieldAliases.length && fieldNames.length != fieldOverrides.length) {
throw new IllegalArgumentException("AtomDataTypeHandler, configured fieldNames, fieldAliases, and fieldOverrides are different lengtsh. " + "Please fix the configuration. " + fieldNames.length + "," + fieldAliases.length + "," + fieldOverrides.length);
}
}
19
View Complete Implementation : ContentIndexingColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Before
public void setUp() throws Exception {
conf = new Configuration();
conf.addResource("config/all-config.xml");
ctx = new TaskAttemptContextImpl(conf, new TaskAttemptID());
ctx.getConfiguration().setInt(ContentIndexingColumnBasedHandler.NUM_SHARDS, 131);
ctx.getConfiguration().set(ContentIndexingColumnBasedHandler.SHARD_TNAME, "shard");
ctx.getConfiguration().set(ContentIndexingColumnBasedHandler.SHARD_GIDX_TNAME, "shardIndex");
ctx.getConfiguration().set(ContentIndexingColumnBasedHandler.SHARD_GRIDX_TNAME, "shardIndex");
ctx.getConfiguration().set(TypeRegistry.INGEST_DATA_TYPES, "test");
ctx.getConfiguration().set("data.name", "test");
ctx.getConfiguration().set("test.data.auth.id.mode", "NEVER");
ctx.getConfiguration().set("test" + BaseIngestHelper.DEFAULT_TYPE, LcNoDiacriticsType.clreplaced.getName());
ctx.getConfiguration().set("test" + TypeRegistry.HANDLER_CLreplacedES, TestContentIndexingColumnBasedHandler.clreplaced.getName());
ctx.getConfiguration().set("test" + TypeRegistry.RAW_READER, TestEventRecordReader.clreplaced.getName());
ctx.getConfiguration().set("test" + TypeRegistry.INGEST_HELPER, TestContentBaseIngestHelper.clreplaced.getName());
// it will die if this field is not faked
ctx.getConfiguration().set(TypeRegistry.EXCLUDED_HANDLER_CLreplacedES, "FAKE_HANDLER_CLreplaced");
helper = new TestContentBaseIngestHelper();
colVis = new ColumnVisibility("");
}
19
View Complete Implementation : AbstractITCommitProtocol.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
protected void replacedertTaskAttemptPathDoesNotExist(AbstractS3ACommitter committer, TaskAttemptContext context) throws IOException {
Path attemptPath = committer.getTaskAttemptPath(context);
ContractTestUtils.replacedertPathDoesNotExist(attemptPath.getFileSystem(context.getConfiguration()), "task attempt dir", attemptPath);
}
18
View Complete Implementation : ContentIndexingColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Test
public void testHandlerNormalizedTokenizedFieldWithSpace() throws Exception {
TypeRegistry.reset();
TypeRegistry.getInstance(ctx.getConfiguration());
setupMocks();
TestContentIndexingColumnBasedHandler handler = new TestContentIndexingColumnBasedHandler();
handler.setup(ctx);
helper.setup(ctx.getConfiguration());
testProcessing(handler, ALPHANUM_LIST, LIST_VALUE_WITH_SPACE, tokenizedExpectedFieldsWithSpace, tokenizedExpectedIndexWithSpace, tokenizedExpectedReverseWithSpace, tokenizedExpectedTfValuesWithSpace, true);
}
18
View Complete Implementation : ContentIndexingColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Test
public void testHandlerListNormalizedAlphanumWithSpace() throws Exception {
ctx.getConfiguration().set("test" + "." + ALPHANUM_LIST + BaseIngestHelper.FIELD_TYPE, LcNoDiacriticsType.clreplaced.getName());
ctx.getConfiguration().set("test" + "." + ContentBaseIngestHelper.LIST_DELIMITERS, LIST_DELIMITERS);
TypeRegistry.reset();
TypeRegistry.getInstance(ctx.getConfiguration());
setupMocks();
TestContentIndexingColumnBasedHandler handler = new TestContentIndexingColumnBasedHandler();
handler.setup(ctx);
helper.setup(ctx.getConfiguration());
testProcessing(handler, ALPHANUM_LIST, LIST_VALUE_WITH_SPACE, listExpectedAlpahnumFields, listExpectedAlpahnumIndex, listExpectedAlpahnumReverse, listExpectedAlphanumTfValues, false);
}
18
View Complete Implementation : DynamicInputChunk.java
Copyright Apache License 2.0
Author : yncxcw
Copyright Apache License 2.0
Author : yncxcw
/**
* Factory method that
* 1. acquires a chunk for the specified map-task attempt
* 2. returns a DynamicInputChunk replacedociated with the acquired chunk-file.
* @param taskAttemptContext The attempt-context for the map task that's
* trying to acquire a chunk.
* @return The acquired dynamic-chunk. The chunk-file is renamed to the
* attempt-id (from the attempt-context.)
* @throws IOException Exception on failure.
* @throws InterruptedException Exception on failure.
*/
public static DynamicInputChunk acquire(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
if (!areInvariantsInitialized())
initializeChunkInvariants(taskAttemptContext.getConfiguration());
String taskId = taskAttemptContext.getTaskAttemptID().getTaskID().toString();
Path acquiredFilePath = new Path(chunkRootPath, taskId);
if (fs.exists(acquiredFilePath)) {
LOG.info("Acquiring pre-replacedigned chunk: " + acquiredFilePath);
return new DynamicInputChunk(acquiredFilePath, taskAttemptContext);
}
for (FileStatus chunkFile : getListOfChunkFiles()) {
if (fs.rename(chunkFile.getPath(), acquiredFilePath)) {
LOG.info(taskId + " acquired " + chunkFile.getPath());
return new DynamicInputChunk(acquiredFilePath, taskAttemptContext);
} else
LOG.warn(taskId + " could not acquire " + chunkFile.getPath());
}
return null;
}
18
View Complete Implementation : JsonRecordReader.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException {
super.initialize(split, context);
if (!(split instanceof FileSplit)) {
throw new IOException("Cannot handle split type " + split.getClreplaced().getName());
}
FileSplit fsplit = (FileSplit) split;
Path file = fsplit.getPath();
rawFileName = file.getName();
fileURI = file.toUri();
FileSystem fs = file.getFileSystem(context.getConfiguration());
InputStream is = fs.open(file);
start = fsplit.getStart();
end = start + fsplit.getLength();
pos = start;
String normURI = fileURI.getScheme() + "://" + fileURI.getPath();
setupReader(is);
if (logger.isInfoEnabled()) {
logger.info("Reading Json records from " + normURI + " via " + is.getClreplaced().getName());
}
jsonHelper = (JsonDataTypeHelper) createHelper(context.getConfiguration());
this.parseHeaderOnly = !jsonHelper.processExtraFields();
jsonFlattener = jsonHelper.newFlattener();
if (logger.isInfoEnabled()) {
logger.info("Json flattener mode: " + jsonFlattener.getFlattenMode().name());
}
}
18
View Complete Implementation : AbstractEventRecordReader.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Override
public void initialize(final InputSplit genericSplit, final TaskAttemptContext context) throws IOException {
initializeEvent(context.getConfiguration());
if (genericSplit instanceof FileSplit) {
final Path p = ((FileSplit) genericSplit).getPath();
final FileSystem sys = p.getFileSystem(context.getConfiguration());
rawFileName = p.toString();
rawFileTimeStamp = sys.getFileStatus(p).getModificationTime();
}
}
18
View Complete Implementation : ContentJsonColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
Copyright Apache License 2.0
Author : NationalSecurityAgency
private JsonRecordReader getJsonRecordReader(String file) throws IOException, URISyntaxException {
InputSplit split = ColumnBasedHandlerTestUtil.getSplit(file);
TaskAttemptContext ctx = new TaskAttemptContextImpl(conf, new TaskAttemptID());
TypeRegistry.reset();
TypeRegistry.getInstance(ctx.getConfiguration());
log.debug(TypeRegistry.getContents());
JsonRecordReader reader = new JsonRecordReader();
reader.initialize(split, ctx);
return reader;
}
18
View Complete Implementation : ContentIndexingColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Test
public void testHandlerListNormalizedAlphanumNoSpace() throws Exception {
ctx.getConfiguration().set("test" + "." + ALPHANUM_LIST + BaseIngestHelper.FIELD_TYPE, LcNoDiacriticsType.clreplaced.getName());
ctx.getConfiguration().set("test" + "." + ContentBaseIngestHelper.LIST_DELIMITERS, LIST_DELIMITERS);
TypeRegistry.reset();
TypeRegistry.getInstance(ctx.getConfiguration());
setupMocks();
TestContentIndexingColumnBasedHandler handler = new TestContentIndexingColumnBasedHandler();
handler.setup(ctx);
helper.setup(ctx.getConfiguration());
testProcessing(handler, ALPHANUM_LIST, LIST_VALUE, listExpectedAlpahnumFields, listExpectedAlpahnumIndex, listExpectedAlpahnumReverse, listExpectedAlphanumTfValues, false);
}
18
View Complete Implementation : ForwardingBigQueryFileOutputFormat.java
Copyright Apache License 2.0
Author : GoogleCloudDataproc
Copyright Apache License 2.0
Author : GoogleCloudDataproc
/**
* Create a new OutputCommitter for this OutputFormat.
*
* @param context the context to create the OutputCommitter from.
* @return the new OutputCommitter for this format.
* @throws IOException if there's an issue while creating the OutputCommitter.
*/
protected OutputCommitter createCommitter(TaskAttemptContext context) throws IOException {
Configuration conf = context.getConfiguration();
return getDelegate(conf).getOutputCommitter(context);
}
18
View Complete Implementation : DynamicFileListRecordReaderTest.java
Copyright Apache License 2.0
Author : GoogleCloudDataproc
Copyright Apache License 2.0
Author : GoogleCloudDataproc
@Before
public void setUp() throws Exception {
MockitoAnnotations.initMocks(this);
LoggerConfig.getConfig(DynamicFileListRecordReader.clreplaced).setLevel(Level.FINE);
// Set up a Configuration which will case "gs://" to grab an InMemoryGoogleHadoopFileSystem.
config = InMemoryGoogleHadoopFileSystem.getSampleConfiguration();
when(mockTaskContext.getConfiguration()).thenReturn(config);
basePath = new Path("gs://foo-bucket/");
shardPath = new Path(basePath, "shard0/data-*.json");
estimatedNumRecords = 2;
fileSystem = basePath.getFileSystem(config);
fileSystem.mkdirs(shardPath.getParent());
// Instead of actually blocking, make our mockSleeper throw an exception that we can catch
// whenever the reader would otherwise be blocking.
doThrow(new RuntimeException(SLEEP_ID)).when(mockSleeper).sleep(anyLong());
resetRecordReader();
}
18
View Complete Implementation : GsonRecordReaderTest.java
Copyright Apache License 2.0
Author : GoogleCloudDataproc
Copyright Apache License 2.0
Author : GoogleCloudDataproc
/**
* Helper function to get GsonRecordReader with multiple records.
*
* @param splitLength the length of the inputSplit in number of chars.
* @throws IOException on IO Error.
*/
public GsonRecordReader getRecordReader(int splitLength) throws Exception {
// Create the task context.
TaskAttemptContext mockJob = Mockito.mock(TaskAttemptContext.clreplaced);
Mockito.when(mockJob.getConfiguration()).thenReturn(config);
Mockito.when(mockJob.getTaskAttemptID()).thenReturn(testTaskAttemptId);
// Write values to file.
Path mockPath = new Path("gs://test_bucket/test-object");
writeFile(ghfs, mockPath, (value1 + "\n" + value2 + "\n").getBytes(UTF_8));
// Create a new InputSplit containing the values.
UnshardedInputSplit inputSplit = new UnshardedInputSplit(mockPath, 0, splitLength, new String[0]);
// Create the GsonRecordReader.
GsonRecordReader reader = new GsonRecordReader();
reader.initialize(inputSplit, mockJob);
return reader;
}
18
View Complete Implementation : ContentIndexingColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Test
public void testHandlerListNormalizedNumericsNoSpace() throws Exception {
ctx.getConfiguration().set("test" + "." + NUMERIC_LIST + BaseIngestHelper.FIELD_TYPE, NumberType.clreplaced.getName());
ctx.getConfiguration().set("test" + "." + ContentBaseIngestHelper.LIST_DELIMITERS, LIST_DELIMITERS);
TypeRegistry.reset();
TypeRegistry.getInstance(ctx.getConfiguration());
setupMocks();
TestContentIndexingColumnBasedHandler handler = new TestContentIndexingColumnBasedHandler();
handler.setup(ctx);
helper.setup(ctx.getConfiguration());
testProcessing(handler, NUMERIC_LIST, LIST_VALUE, listExpectedNumericFields, listExpectedNumericIndex, listExpectedNumericReverse, listExpectedNumericTfValues, false);
}
18
View Complete Implementation : ContentIndexingColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Test
public void testHandlerNormalizedTokenizedFieldNoSpace() throws Exception {
TypeRegistry.reset();
TypeRegistry.getInstance(ctx.getConfiguration());
setupMocks();
TestContentIndexingColumnBasedHandler handler = new TestContentIndexingColumnBasedHandler();
handler.setup(ctx);
helper.setup(ctx.getConfiguration());
testProcessing(handler, ALPHANUM_LIST, LIST_VALUE, tokenizedExpectedFields, tokenizedExpectedIndex, tokenizedExpectedReverse, tokenizedExpectedTfValues, true);
}
18
View Complete Implementation : AbstractColumnBasedHandler.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Override
public void setup(TaskAttemptContext context) {
super.setup(context);
TypeRegistry registry = TypeRegistry.getInstance(context.getConfiguration());
Type type = registry.get(context.getConfiguration().get(DataTypeHelper.Properties.DATA_NAME));
helper = type.getIngestHelper(context.getConfiguration());
log.info(this.getClreplaced().getSimpleName() + " configured.");
}
18
View Complete Implementation : ContentCSVIndexingColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
Copyright Apache License 2.0
Author : NationalSecurityAgency
private CSVRecordReader getCSVRecordReader(String file) throws IOException, URISyntaxException {
InputSplit split = ColumnBasedHandlerTestUtil.getSplit(file);
TaskAttemptContext ctx = new TaskAttemptContextImpl(conf, new TaskAttemptID());
TypeRegistry.reset();
TypeRegistry.getInstance(ctx.getConfiguration());
log.debug(TypeRegistry.getContents());
CSVRecordReader reader = new CSVRecordReader();
reader.initialize(split, ctx);
return reader;
}
18
View Complete Implementation : ContentIndexingColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Test
public void testHandlerListNormalizedAlphanumWithEmptyEntry() throws Exception {
ctx.getConfiguration().set("test" + "." + ALPHANUM_LIST + BaseIngestHelper.FIELD_TYPE, LcNoDiacriticsType.clreplaced.getName());
ctx.getConfiguration().set("test" + "." + ContentBaseIngestHelper.LIST_DELIMITERS, LIST_DELIMITERS);
TypeRegistry.reset();
TypeRegistry.getInstance(ctx.getConfiguration());
setupMocks();
TestContentIndexingColumnBasedHandler handler = new TestContentIndexingColumnBasedHandler();
handler.setup(ctx);
helper.setup(ctx.getConfiguration());
testProcessing(handler, ALPHANUM_LIST, LIST_VALUE_WITH_EMPTY_ENTRY, listExpectedAlpahnumFields, listExpectedAlpahnumIndex, listExpectedAlpahnumReverse, listExpectedAlphanumTfValues, false);
}
18
View Complete Implementation : StatsDHelper.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
Copyright Apache License 2.0
Author : NationalSecurityAgency
public void setup(TaskAttemptContext context) {
setup(context.getConfiguration());
}
18
View Complete Implementation : ContentIndexingColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Test
public void testHandlerListNormalizedNumericWithSpace() throws Exception {
ctx.getConfiguration().set("test" + "." + NUMERIC_LIST + BaseIngestHelper.FIELD_TYPE, NumberType.clreplaced.getName());
ctx.getConfiguration().set("test" + "." + ContentBaseIngestHelper.LIST_DELIMITERS, LIST_DELIMITERS);
TypeRegistry.reset();
TypeRegistry.getInstance(ctx.getConfiguration());
setupMocks();
TestContentIndexingColumnBasedHandler handler = new TestContentIndexingColumnBasedHandler();
handler.setup(ctx);
helper.setup(ctx.getConfiguration());
testProcessing(handler, NUMERIC_LIST, LIST_VALUE_WITH_SPACE, listExpectedNumericFields, listExpectedNumericIndex, listExpectedNumericReverse, listExpectedNumericTfValues, false);
}
18
View Complete Implementation : TransformOutputFormat.java
Copyright Apache License 2.0
Author : marklogic
Copyright Apache License 2.0
Author : marklogic
@Override
public RecordWriter<DoreplacedentURI, VALUEOUT> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
fastLoad = Boolean.valueOf(conf.get(OUTPUT_FAST_LOAD));
Map<String, ContentSource> sourceMap = getSourceMap(fastLoad, context);
getMimetypesMap();
// construct the ContentWriter
return new TransformWriter<VALUEOUT>(conf, sourceMap, fastLoad, am);
}
17
View Complete Implementation : FileOutputCommitter.java
Copyright Apache License 2.0
Author : yncxcw
Copyright Apache License 2.0
Author : yncxcw
@Private
public boolean needsTaskCommit(TaskAttemptContext context, Path taskAttemptPath) throws IOException {
if (hasOutputPath()) {
if (taskAttemptPath == null) {
taskAttemptPath = getTaskAttemptPath(context);
}
FileSystem fs = taskAttemptPath.getFileSystem(context.getConfiguration());
return fs.exists(taskAttemptPath);
}
return false;
}
17
View Complete Implementation : FileOutputCommitter.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Private
public void abortTask(TaskAttemptContext context, Path taskAttemptPath) throws IOException {
if (hasOutputPath()) {
context.progress();
if (taskAttemptPath == null) {
taskAttemptPath = getTaskAttemptPath(context);
}
FileSystem fs = taskAttemptPath.getFileSystem(context.getConfiguration());
if (!fs.delete(taskAttemptPath, true)) {
LOG.warn("Could not delete " + taskAttemptPath);
}
} else {
LOG.warn("Output Path is null in abortTask()");
}
}
17
View Complete Implementation : MapFileOutputFormat.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public RecordWriter<WritableComparable<?>, Writable> getRecordWriter(TaskAttemptContext context) throws IOException {
Configuration conf = context.getConfiguration();
CompressionCodec codec = null;
CompressionType compressionType = CompressionType.NONE;
if (getCompressOutput(context)) {
// find the kind of compression to do
compressionType = SequenceFileOutputFormat.getOutputCompressionType(context);
// find the right codec
Clreplaced<?> codecClreplaced = getOutputCompressorClreplaced(context, DefaultCodec.clreplaced);
codec = (CompressionCodec) ReflectionUtils.newInstance(codecClreplaced, conf);
}
Path file = getDefaultWorkFile(context, "");
FileSystem fs = file.getFileSystem(conf);
// ignore the progress parameter, since MapFile is local
final MapFile.Writer out = new MapFile.Writer(conf, fs, file.toString(), context.getOutputKeyClreplaced().replacedubclreplaced(WritableComparable.clreplaced), context.getOutputValueClreplaced().replacedubclreplaced(Writable.clreplaced), compressionType, codec, context);
return new RecordWriter<WritableComparable<?>, Writable>() {
public void write(WritableComparable<?> key, Writable value) throws IOException {
out.append(key, value);
}
public void close(TaskAttemptContext context) throws IOException {
out.close();
}
};
}
17
View Complete Implementation : CompositeInputFormat.java
Copyright Apache License 2.0
Author : yncxcw
Copyright Apache License 2.0
Author : yncxcw
/**
* Construct a CompositeRecordReader for the children of this InputFormat
* as defined in the init expression.
* The outermost join need only be composable, not necessarily a composite.
* Mandating TupleWritable isn't strictly correct.
*/
// child types unknown
@SuppressWarnings("unchecked")
public RecordReader<K, TupleWritable> createRecordReader(InputSplit split, TaskAttemptContext taskContext) throws IOException, InterruptedException {
setFormat(taskContext.getConfiguration());
return root.createRecordReader(split, taskContext);
}
17
View Complete Implementation : FileOutputCommitter.java
Copyright Apache License 2.0
Author : yncxcw
Copyright Apache License 2.0
Author : yncxcw
@Private
public void abortTask(TaskAttemptContext context, Path taskAttemptPath) throws IOException {
if (hasOutputPath()) {
context.progress();
if (taskAttemptPath == null) {
taskAttemptPath = getTaskAttemptPath(context);
}
FileSystem fs = taskAttemptPath.getFileSystem(context.getConfiguration());
if (!fs.delete(taskAttemptPath, true)) {
LOG.warn("Could not delete " + taskAttemptPath);
}
} else {
LOG.warn("Output Path is null in abortTask()");
}
}
17
View Complete Implementation : CompositeInputFormat.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* Construct a CompositeRecordReader for the children of this InputFormat
* as defined in the init expression.
* The outermost join need only be composable, not necessarily a composite.
* Mandating TupleWritable isn't strictly correct.
*/
// child types unknown
@SuppressWarnings("unchecked")
public RecordReader<K, TupleWritable> createRecordReader(InputSplit split, TaskAttemptContext taskContext) throws IOException, InterruptedException {
setFormat(taskContext.getConfiguration());
return root.createRecordReader(split, taskContext);
}
17
View Complete Implementation : MapFileOutputFormat.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
public RecordWriter<WritableComparable<?>, Writable> getRecordWriter(TaskAttemptContext context) throws IOException {
Configuration conf = context.getConfiguration();
CompressionCodec codec = null;
CompressionType compressionType = CompressionType.NONE;
if (getCompressOutput(context)) {
// find the kind of compression to do
compressionType = SequenceFileOutputFormat.getOutputCompressionType(context);
// find the right codec
Clreplaced<?> codecClreplaced = getOutputCompressorClreplaced(context, DefaultCodec.clreplaced);
codec = (CompressionCodec) ReflectionUtils.newInstance(codecClreplaced, conf);
}
Path file = getDefaultWorkFile(context, "");
FileSystem fs = file.getFileSystem(conf);
// ignore the progress parameter, since MapFile is local
final MapFile.Writer out = new MapFile.Writer(conf, fs, file.toString(), context.getOutputKeyClreplaced().replacedubclreplaced(WritableComparable.clreplaced), context.getOutputValueClreplaced().replacedubclreplaced(Writable.clreplaced), compressionType, codec, context);
return new RecordWriter<WritableComparable<?>, Writable>() {
public void write(WritableComparable<?> key, Writable value) throws IOException {
out.append(key, value);
}
public void close(TaskAttemptContext context) throws IOException {
out.close();
}
};
}
17
View Complete Implementation : FileOutputCommitter.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Private
public boolean needsTaskCommit(TaskAttemptContext context, Path taskAttemptPath) throws IOException {
if (hasOutputPath()) {
if (taskAttemptPath == null) {
taskAttemptPath = getTaskAttemptPath(context);
}
FileSystem fs = taskAttemptPath.getFileSystem(context.getConfiguration());
return fs.exists(taskAttemptPath);
}
return false;
}
17
View Complete Implementation : FixedLengthRecordReader.java
Copyright Apache License 2.0
Author : yncxcw
Copyright Apache License 2.0
Author : yncxcw
@Override
public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
FileSplit split = (FileSplit) genericSplit;
Configuration job = context.getConfiguration();
final Path file = split.getPath();
initialize(job, split.getStart(), split.getLength(), file);
}
17
View Complete Implementation : ForwardingBigQueryFileOutputFormatTest.java
Copyright Apache License 2.0
Author : GoogleCloudDataproc
Copyright Apache License 2.0
Author : GoogleCloudDataproc
/**
* Verify exceptions are being thrown.
*/
/**
* Sets up common objects for testing before each test.
*/
@Before
public void setUp() throws Exception {
// Generate Mocks.
MockitoAnnotations.initMocks(this);
// Create the file system.
ghfs = new InMemoryGoogleHadoopFileSystem();
// Create the configuration, but setup in the tests.
job = Job.getInstance(InMemoryGoogleHadoopFileSystem.getSampleConfiguration());
conf = job.getConfiguration();
CredentialConfigurationUtil.addTestConfigurationSettings(conf);
BigQueryOutputConfiguration.configureWithAutoSchema(conf, QUALIFIED_TEST_TABLE_ID, TEST_OUTPUT_PATH_STRING, TEST_FILE_FORMAT, TEST_OUTPUT_CLreplaced);
// Configure mocks.
when(mockTaskAttemptContext.getConfiguration()).thenReturn(conf);
when(mockTaskAttemptContext.getTaskAttemptID()).thenReturn(TEST_TASK_ATTEMPT_ID);
when(mockFileOutputFormat.getOutputCommitter(eq(mockTaskAttemptContext))).thenReturn(mockOutputCommitter);
when(mockFileOutputFormat.getRecordWriter(eq(mockTaskAttemptContext))).thenReturn(mockRecordWriter);
// Create and setup the output format.
outputFormat = new ForwardingBigQueryFileOutputFormat<Text, Text>();
outputFormat.setDelegate(mockFileOutputFormat);
}
17
View Complete Implementation : PathOutputCommitterFactory.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Create the committer factory for a task attempt and destination, then
* create the committer from it.
* @param outputPath the task's output path, or or null if no output path
* has been defined.
* @param context the task attempt context
* @return the committer to use
* @throws IOException problems instantiating the committer
*/
public static PathOutputCommitter createCommitter(Path outputPath, TaskAttemptContext context) throws IOException {
return getCommitterFactory(outputPath, context.getConfiguration()).createOutputCommitter(outputPath, context);
}
17
View Complete Implementation : LazyOutputFormat.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Override
public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException {
if (baseOut == null) {
getBaseOutputFormat(context.getConfiguration());
}
return super.getOutputCommitter(context);
}
17
View Complete Implementation : FederatedBigQueryOutputCommitterTest.java
Copyright Apache License 2.0
Author : GoogleCloudDataproc
Copyright Apache License 2.0
Author : GoogleCloudDataproc
// Verify exceptions are being thrown.
// Sets up common objects for testing before each test.
@Before
public void setUp() throws IOException {
// Generate Mocks.
MockitoAnnotations.initMocks(this);
// Create the file system.
ghfs = new InMemoryGoogleHadoopFileSystem();
// Setup the configuration.
job = Job.getInstance(InMemoryGoogleHadoopFileSystem.getSampleConfiguration());
conf = job.getConfiguration();
CredentialConfigurationUtil.addTestConfigurationSettings(conf);
BigQueryOutputConfiguration.configure(conf, QUALIFIED_TEST_TABLE_ID, TEST_TABLE_SCHEMA, TEST_OUTPUT_PATH_STRING, TEST_FILE_FORMAT, TEST_OUTPUT_CLreplaced);
// Setup sample data.
outputTableRef = BigQueryOutputConfiguration.getTableReference(conf);
outputPath = BigQueryOutputConfiguration.getGcsOutputPath(conf);
outputSampleFilePath = new Path(TEST_OUTPUT_FILE_STRING);
// Configure mocks.
when(mockTaskAttemptContext.getConfiguration()).thenReturn(conf);
when(mockTaskAttemptContext.getTaskAttemptID()).thenReturn(TEST_TASK_ATTEMPT_ID);
// Setup committer.
committer = new FederatedBigQueryOutputCommitter(mockTaskAttemptContext, mockCommitter);
committer.setBigQueryHelper(mockBigQueryHelper);
}
17
View Complete Implementation : DBInputFormat.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* {@inheritDoc}
*/
public RecordReader<LongWritable, T> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
return createDBRecordReader((DBInputSplit) split, context.getConfiguration());
}
17
View Complete Implementation : CompositeInputFormat.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Construct a CompositeRecordReader for the children of this InputFormat
* as defined in the init expression.
* The outermost join need only be composable, not necessarily a composite.
* Mandating TupleWritable isn't strictly correct.
*/
// child types unknown
@SuppressWarnings("unchecked")
public RecordReader<K, TupleWritable> createRecordReader(InputSplit split, TaskAttemptContext taskContext) throws IOException, InterruptedException {
setFormat(taskContext.getConfiguration());
return root.createRecordReader(split, taskContext);
}
17
View Complete Implementation : DBInputFormat.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* {@inheritDoc}
*/
public RecordReader<LongWritable, T> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
return createDBRecordReader((DBInputSplit) split, context.getConfiguration());
}
17
View Complete Implementation : NYCTLCColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
Copyright Apache License 2.0
Author : NationalSecurityAgency
private NYCTLCReader getNYCTLCRecordReader(String file) throws IOException, URISyntaxException {
InputSplit split = ColumnBasedHandlerTestUtil.getSplit(file);
TaskAttemptContext ctx = new TaskAttemptContextImpl(conf, new TaskAttemptID());
TypeRegistry.reset();
TypeRegistry.getInstance(ctx.getConfiguration());
log.debug(TypeRegistry.getContents());
NYCTLCReader reader = new NYCTLCReader();
reader.initialize(split, ctx);
return reader;
}
17
View Complete Implementation : FixedLengthRecordReader.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Override
public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
FileSplit split = (FileSplit) genericSplit;
Configuration job = context.getConfiguration();
final Path file = split.getPath();
initialize(job, split.getStart(), split.getLength(), file);
}
17
View Complete Implementation : FileOutputCommitter.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Private
public boolean needsTaskCommit(TaskAttemptContext context, Path taskAttemptPath) throws IOException {
if (hasOutputPath()) {
if (taskAttemptPath == null) {
taskAttemptPath = getTaskAttemptPath(context);
}
FileSystem fs = taskAttemptPath.getFileSystem(context.getConfiguration());
return fs.exists(taskAttemptPath);
}
return false;
}
17
View Complete Implementation : FileOutputCommitter.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Private
public void abortTask(TaskAttemptContext context, Path taskAttemptPath) throws IOException {
if (hasOutputPath()) {
context.progress();
if (taskAttemptPath == null) {
taskAttemptPath = getTaskAttemptPath(context);
}
FileSystem fs = taskAttemptPath.getFileSystem(context.getConfiguration());
if (!fs.delete(taskAttemptPath, true)) {
LOG.warn("Could not delete " + taskAttemptPath);
}
} else {
LOG.warn("Output Path is null in abortTask()");
}
}
17
View Complete Implementation : LazyOutputFormat.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Override
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
if (baseOut == null) {
getBaseOutputFormat(context.getConfiguration());
}
return new LazyRecordWriter<K, V>(baseOut, context);
}