org.apache.hadoop.mapreduce.TaskAttemptContext.getConfiguration() - java examples

Here are the examples of the java api org.apache.hadoop.mapreduce.TaskAttemptContext.getConfiguration() taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

155 Examples 7

19 View Complete Implementation : MagicS3GuardCommitter.java
Copyright Apache License 2.0
Author : apache
/**
 * Did this task write any files in the work directory?
 * Probes for a task existing by looking to see if the attempt dir exists.
 * This adds more HTTP requests to the call. It may be better just to
 * return true and rely on the commit task doing the work.
 * @param context the task's context
 * @return true if the attempt path exists
 * @throws IOException failure to list the path
 */
@Override
public boolean needsTaskCommit(TaskAttemptContext context) throws IOException {
    Path taskAttemptPath = getTaskAttemptPath(context);
    try (DurationInfo d = new DurationInfo(LOG, "needsTaskCommit task %s", context.getTaskAttemptID())) {
        return taskAttemptPath.getFileSystem(context.getConfiguration()).exists(taskAttemptPath);
    }
}

19 View Complete Implementation : WikipediaDataTypeHandler.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Override
public void setup(TaskAttemptContext context) {
    super.setup(context);
    this.ingestHelper = (WikipediaIngestHelper) this.getHelper(null);
    this.helper = this.ingestHelper.getDataTypeHelper();
    try {
        this.parser = factory.newDoreplacedentBuilder();
    } catch (ParserConfigurationException e) {
        throw new RuntimeException("Error instantiating DoreplacedentBuilder", e);
    }
    Configuration conf = context.getConfiguration();
    this.counters = new ContentIndexCounters();
    disableDCol = conf.getBoolean(OPT_NO_D_COL, disableDCol);
    docWriterService = Executors.newSingleThreadExecutor();
    try {
        AcreplaceduloHelper acreplaceduloHelper = new AcreplaceduloHelper();
        acreplaceduloHelper.setup(conf);
        log.debug("Attempting to create Acreplacedulo connection.");
        docWriter = acreplaceduloHelper.getConnector().createBatchWriter(conf.get("shard.table.name"), new BatchWriterConfig().setMaxLatency(60, TimeUnit.SECONDS).setMaxMemory(100000000L).setMaxWriteThreads(10));
        log.debug("Created connection to Acreplacedulo for asynchronous doreplacedent storage.");
    } catch (Exception e) {
        log.warn("No doreplacedent payloads will be written to Acreplacedulo.", e);
        // giving a stub batchwriter means I don't have a bunch of "if(writer != null)"s lying around
        docWriter = new BatchWriter() {

            @Override
            public void addMutation(Mutation m) {
            }

            @Override
            public void addMutations(Iterable<Mutation> iterable) throws MutationsRejectedException {
            }

            @Override
            public void flush() throws MutationsRejectedException {
            }

            @Override
            public void close() throws MutationsRejectedException {
            }
        };
    }
}

19 View Complete Implementation : DelimitedTextInputFormat.java
Copyright Apache License 2.0
Author : marklogic
@Override
public RecordReader<DoreplacedentURIWithSourceInfo, Text> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    if (isSplitInput(context.getConfiguration())) {
        return new SplitDelimitedTextReader<Text>();
    } else {
        return new DelimitedTextReader<Text>();
    }
}

19 View Complete Implementation : ContentOutputFormat.java
Copyright Apache License 2.0
Author : marklogic
@Override
public RecordWriter<DoreplacedentURI, VALUEOUT> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    // TODO: if MAPREDUCE-3377 still exists, need to re-run initialize
    fastLoad = Boolean.valueOf(conf.get(OUTPUT_FAST_LOAD));
    Map<String, ContentSource> sourceMap = getSourceMap(fastLoad, context);
    // construct the ContentWriter
    return new ContentWriter<VALUEOUT>(conf, sourceMap, fastLoad, am);
}

19 View Complete Implementation : StagingCommitter.java
Copyright Apache License 2.0
Author : apache
/**
 * Get the work path for a task.
 * @param context job/task complex
 * @param uuid UUID
 * @return a path
 * @throws IOException failure to build the path
 */
private static Path taskAttemptWorkingPath(TaskAttemptContext context, String uuid) throws IOException {
    return getTaskAttemptPath(context, Paths.getLocalTaskAttemptTempDir(context.getConfiguration(), uuid, context.getTaskAttemptID()));
}

19 View Complete Implementation : GeoWaveDataTypeHandler.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Override
public void setup(final TaskAttemptContext context) {
    final TypeRegistry registry = TypeRegistry.getInstance(context.getConfiguration());
    final Type type = registry.get(context.getConfiguration().get(DataTypeHelperImpl.Properties.DATA_NAME));
    conf = context.getConfiguration();
    final String sftName = conf.get(type.typeName() + GEOWAVE_FEATURE_TYPE_NAME);
    final String sftSpec = conf.get(type.typeName() + GEOWAVE_FEATURE_TYPE_SPEC);
    try {
        originalType = (sftSpec != null) ? DataUtilities.createType(sftName, sftSpec) : buildSimpleFeatureType(type.typeName());
    } catch (SchemaException e) {
        log.error("Unable to parse Simple Feature Type from spec [" + sftSpec + "]");
    }
    // create a simple feature data adapter
    dataAdapter = new FeatureDataAdapter(originalType);
    // determine whether features need to be reprojected
    if (!GeoWaveGTDataStore.DEFAULT_CRS.equals(originalType.getCoordinateReferenceSystem())) {
        reprojectedType = SimpleFeatureTypeBuilder.retype(originalType, GeoWaveGTDataStore.DEFAULT_CRS);
        if (originalType.getCoordinateReferenceSystem() != null) {
            try {
                transform = CRS.findMathTransform(originalType.getCoordinateReferenceSystem(), GeoWaveGTDataStore.DEFAULT_CRS, true);
            } catch (final FactoryException e) {
                log.warn("Unable to create coordinate reference system transform", e);
            }
        }
    }
    // features are built using the original projection
    builder = new SimpleFeatureBuilder(originalType);
    // create an index
    index = initIndex(conf.get(type.typeName() + GEOWAVE_INDEX_TYPE, null));
    // namespace
    tableNamespace = conf.get(type.typeName() + GEOWAVE_NAMESPACE, null);
    // flatten geometry
    flattenGeometry = conf.getBoolean(type.typeName() + GEOWAVE_FLATTEN_GEOMETRY, flattenGeometry);
    // GeoWave metadata
    metadata = new GeoWaveMetadata(tableNamespace, index, dataAdapter, originalType, reprojectedType, transform, flattenGeometry);
}

19 View Complete Implementation : AggregateXMLReader.java
Copyright Apache License 2.0
Author : marklogic
protected void initAggConf(TaskAttemptContext context) {
    Configuration conf = context.getConfiguration();
    idName = conf.get(ConfigConstants.CONF_INPUT_URI_ID);
    if (idName == null) {
        useAutomaticId = true;
    }
    recordName = conf.get(ConfigConstants.CONF_AGGREGATE_RECORD_ELEMENT);
    recordNamespace = conf.get(ConfigConstants.CONF_AGGREGATE_RECORD_NAMESPACE);
}

19 View Complete Implementation : MagicS3GuardCommitter.java
Copyright Apache License 2.0
Author : apache
/**
 * Abort a task. Attempt load then abort all pending files,
 * then try to delete the task attempt path.
 * This method may be called on the job committer, rather than the
 * task one (such as in the MapReduce AM after a task container failure).
 * It must extract all paths and state from the preplaceded in context.
 * @param context task context
 * @throws IOException if there was some problem querying the path other
 * than it not actually existing.
 */
@Override
public void abortTask(TaskAttemptContext context) throws IOException {
    Path attemptPath = getTaskAttemptPath(context);
    try (DurationInfo d = new DurationInfo(LOG, "Abort task %s", context.getTaskAttemptID())) {
        getCommitOperations().abortAllSinglePendingCommits(attemptPath, true);
    } finally {
        deleteQuietly(attemptPath.getFileSystem(context.getConfiguration()), attemptPath, true);
        destroyThreadPool();
    }
}

19 View Complete Implementation : AtomDataTypeHandler.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Override
public void setup(TaskAttemptContext context) {
    conf = context.getConfiguration();
    tableName = ConfigurationHelper.isNull(context.getConfiguration(), ATOM_TABLE_NAME, String.clreplaced);
    categoryTableName = tableName + "Categories";
    subCategories = new HashMap<>();
    markingFunctions = MarkingFunctions.Factory.createMarkingFunctions();
    TypeRegistry.getInstance(context.getConfiguration());
    String[] types = ConfigurationHelper.isNull(context.getConfiguration(), ATOM_TYPES_TO_PROCESS, String[].clreplaced);
    // Set up the ingest helpers for the known datatypes.
    fieldNames = ConfigurationHelper.isNull(context.getConfiguration(), ATOM_FIELD_NAMES, String[].clreplaced);
    // Configuration.getStrings() eats empty values, we don't want to do that. Split it ourselves.
    String aliases = ConfigurationHelper.isNull(context.getConfiguration(), ATOM_FIELD_ALIASES, String.clreplaced);
    // keeps empty elements
    fieldAliases = StringUtils.split(aliases, ',', true);
    String overrides = ConfigurationHelper.isNull(context.getConfiguration(), ATOM_FIELD_VALUE_OVERRIDES, String.clreplaced);
    // keeps empty elements
    fieldOverrides = StringUtils.split(overrides, ',', true);
    sCategories = StringUtils.split(ConfigurationHelper.isNull(context.getConfiguration(), ATOM_CATEGORY_SUB_FIELD, String.clreplaced), ',', false);
    Set<String> tSet;
    for (String s : sCategories) {
        String[] field_value = StringUtils.split(s, ':', false);
        if (field_value.length == 2 && (!Strings.isNullOrEmpty(field_value[0]) && !Strings.isNullOrEmpty(field_value[1]))) {
            if (!subCategories.containsKey(field_value[0])) {
                tSet = new HashSet<>();
            } else {
                tSet = subCategories.get(field_value[0]);
            }
            System.err.println("Value: " + field_value[0] + " " + field_value[1]);
            tSet.add(field_value[1]);
            subCategories.put(field_value[0], tSet);
        }
    }
    // Make sure these 3 arrays are all the same size.
    if (fieldNames.length != fieldAliases.length && fieldNames.length != fieldOverrides.length) {
        throw new IllegalArgumentException("AtomDataTypeHandler, configured fieldNames, fieldAliases, and fieldOverrides are different lengtsh.  " + "Please fix the configuration. " + fieldNames.length + "," + fieldAliases.length + "," + fieldOverrides.length);
    }
}

19 View Complete Implementation : ContentIndexingColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Before
public void setUp() throws Exception {
    conf = new Configuration();
    conf.addResource("config/all-config.xml");
    ctx = new TaskAttemptContextImpl(conf, new TaskAttemptID());
    ctx.getConfiguration().setInt(ContentIndexingColumnBasedHandler.NUM_SHARDS, 131);
    ctx.getConfiguration().set(ContentIndexingColumnBasedHandler.SHARD_TNAME, "shard");
    ctx.getConfiguration().set(ContentIndexingColumnBasedHandler.SHARD_GIDX_TNAME, "shardIndex");
    ctx.getConfiguration().set(ContentIndexingColumnBasedHandler.SHARD_GRIDX_TNAME, "shardIndex");
    ctx.getConfiguration().set(TypeRegistry.INGEST_DATA_TYPES, "test");
    ctx.getConfiguration().set("data.name", "test");
    ctx.getConfiguration().set("test.data.auth.id.mode", "NEVER");
    ctx.getConfiguration().set("test" + BaseIngestHelper.DEFAULT_TYPE, LcNoDiacriticsType.clreplaced.getName());
    ctx.getConfiguration().set("test" + TypeRegistry.HANDLER_CLreplacedES, TestContentIndexingColumnBasedHandler.clreplaced.getName());
    ctx.getConfiguration().set("test" + TypeRegistry.RAW_READER, TestEventRecordReader.clreplaced.getName());
    ctx.getConfiguration().set("test" + TypeRegistry.INGEST_HELPER, TestContentBaseIngestHelper.clreplaced.getName());
    // it will die if this field is not faked
    ctx.getConfiguration().set(TypeRegistry.EXCLUDED_HANDLER_CLreplacedES, "FAKE_HANDLER_CLreplaced");
    helper = new TestContentBaseIngestHelper();
    colVis = new ColumnVisibility("");
}

19 View Complete Implementation : AbstractITCommitProtocol.java
Copyright Apache License 2.0
Author : apache
protected void replacedertTaskAttemptPathDoesNotExist(AbstractS3ACommitter committer, TaskAttemptContext context) throws IOException {
    Path attemptPath = committer.getTaskAttemptPath(context);
    ContractTestUtils.replacedertPathDoesNotExist(attemptPath.getFileSystem(context.getConfiguration()), "task attempt dir", attemptPath);
}

18 View Complete Implementation : ContentIndexingColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Test
public void testHandlerNormalizedTokenizedFieldWithSpace() throws Exception {
    TypeRegistry.reset();
    TypeRegistry.getInstance(ctx.getConfiguration());
    setupMocks();
    TestContentIndexingColumnBasedHandler handler = new TestContentIndexingColumnBasedHandler();
    handler.setup(ctx);
    helper.setup(ctx.getConfiguration());
    testProcessing(handler, ALPHANUM_LIST, LIST_VALUE_WITH_SPACE, tokenizedExpectedFieldsWithSpace, tokenizedExpectedIndexWithSpace, tokenizedExpectedReverseWithSpace, tokenizedExpectedTfValuesWithSpace, true);
}

18 View Complete Implementation : ContentIndexingColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Test
public void testHandlerListNormalizedAlphanumWithSpace() throws Exception {
    ctx.getConfiguration().set("test" + "." + ALPHANUM_LIST + BaseIngestHelper.FIELD_TYPE, LcNoDiacriticsType.clreplaced.getName());
    ctx.getConfiguration().set("test" + "." + ContentBaseIngestHelper.LIST_DELIMITERS, LIST_DELIMITERS);
    TypeRegistry.reset();
    TypeRegistry.getInstance(ctx.getConfiguration());
    setupMocks();
    TestContentIndexingColumnBasedHandler handler = new TestContentIndexingColumnBasedHandler();
    handler.setup(ctx);
    helper.setup(ctx.getConfiguration());
    testProcessing(handler, ALPHANUM_LIST, LIST_VALUE_WITH_SPACE, listExpectedAlpahnumFields, listExpectedAlpahnumIndex, listExpectedAlpahnumReverse, listExpectedAlphanumTfValues, false);
}

18 View Complete Implementation : DynamicInputChunk.java
Copyright Apache License 2.0
Author : yncxcw
/**
 * Factory method that
 * 1. acquires a chunk for the specified map-task attempt
 * 2. returns a DynamicInputChunk replacedociated with the acquired chunk-file.
 * @param taskAttemptContext The attempt-context for the map task that's
 * trying to acquire a chunk.
 * @return The acquired dynamic-chunk. The chunk-file is renamed to the
 * attempt-id (from the attempt-context.)
 * @throws IOException Exception on failure.
 * @throws InterruptedException Exception on failure.
 */
public static DynamicInputChunk acquire(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
    if (!areInvariantsInitialized())
        initializeChunkInvariants(taskAttemptContext.getConfiguration());
    String taskId = taskAttemptContext.getTaskAttemptID().getTaskID().toString();
    Path acquiredFilePath = new Path(chunkRootPath, taskId);
    if (fs.exists(acquiredFilePath)) {
        LOG.info("Acquiring pre-replacedigned chunk: " + acquiredFilePath);
        return new DynamicInputChunk(acquiredFilePath, taskAttemptContext);
    }
    for (FileStatus chunkFile : getListOfChunkFiles()) {
        if (fs.rename(chunkFile.getPath(), acquiredFilePath)) {
            LOG.info(taskId + " acquired " + chunkFile.getPath());
            return new DynamicInputChunk(acquiredFilePath, taskAttemptContext);
        } else
            LOG.warn(taskId + " could not acquire " + chunkFile.getPath());
    }
    return null;
}

18 View Complete Implementation : JsonRecordReader.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException {
    super.initialize(split, context);
    if (!(split instanceof FileSplit)) {
        throw new IOException("Cannot handle split type " + split.getClreplaced().getName());
    }
    FileSplit fsplit = (FileSplit) split;
    Path file = fsplit.getPath();
    rawFileName = file.getName();
    fileURI = file.toUri();
    FileSystem fs = file.getFileSystem(context.getConfiguration());
    InputStream is = fs.open(file);
    start = fsplit.getStart();
    end = start + fsplit.getLength();
    pos = start;
    String normURI = fileURI.getScheme() + "://" + fileURI.getPath();
    setupReader(is);
    if (logger.isInfoEnabled()) {
        logger.info("Reading Json records from " + normURI + " via " + is.getClreplaced().getName());
    }
    jsonHelper = (JsonDataTypeHelper) createHelper(context.getConfiguration());
    this.parseHeaderOnly = !jsonHelper.processExtraFields();
    jsonFlattener = jsonHelper.newFlattener();
    if (logger.isInfoEnabled()) {
        logger.info("Json flattener mode: " + jsonFlattener.getFlattenMode().name());
    }
}

18 View Complete Implementation : AbstractEventRecordReader.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Override
public void initialize(final InputSplit genericSplit, final TaskAttemptContext context) throws IOException {
    initializeEvent(context.getConfiguration());
    if (genericSplit instanceof FileSplit) {
        final Path p = ((FileSplit) genericSplit).getPath();
        final FileSystem sys = p.getFileSystem(context.getConfiguration());
        rawFileName = p.toString();
        rawFileTimeStamp = sys.getFileStatus(p).getModificationTime();
    }
}

18 View Complete Implementation : ContentJsonColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
private JsonRecordReader getJsonRecordReader(String file) throws IOException, URISyntaxException {
    InputSplit split = ColumnBasedHandlerTestUtil.getSplit(file);
    TaskAttemptContext ctx = new TaskAttemptContextImpl(conf, new TaskAttemptID());
    TypeRegistry.reset();
    TypeRegistry.getInstance(ctx.getConfiguration());
    log.debug(TypeRegistry.getContents());
    JsonRecordReader reader = new JsonRecordReader();
    reader.initialize(split, ctx);
    return reader;
}

18 View Complete Implementation : ContentIndexingColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Test
public void testHandlerListNormalizedAlphanumNoSpace() throws Exception {
    ctx.getConfiguration().set("test" + "." + ALPHANUM_LIST + BaseIngestHelper.FIELD_TYPE, LcNoDiacriticsType.clreplaced.getName());
    ctx.getConfiguration().set("test" + "." + ContentBaseIngestHelper.LIST_DELIMITERS, LIST_DELIMITERS);
    TypeRegistry.reset();
    TypeRegistry.getInstance(ctx.getConfiguration());
    setupMocks();
    TestContentIndexingColumnBasedHandler handler = new TestContentIndexingColumnBasedHandler();
    handler.setup(ctx);
    helper.setup(ctx.getConfiguration());
    testProcessing(handler, ALPHANUM_LIST, LIST_VALUE, listExpectedAlpahnumFields, listExpectedAlpahnumIndex, listExpectedAlpahnumReverse, listExpectedAlphanumTfValues, false);
}

18 View Complete Implementation : ForwardingBigQueryFileOutputFormat.java
Copyright Apache License 2.0
Author : GoogleCloudDataproc
/**
 * Create a new OutputCommitter for this OutputFormat.
 *
 * @param context the context to create the OutputCommitter from.
 * @return the new OutputCommitter for this format.
 * @throws IOException if there's an issue while creating the OutputCommitter.
 */
protected OutputCommitter createCommitter(TaskAttemptContext context) throws IOException {
    Configuration conf = context.getConfiguration();
    return getDelegate(conf).getOutputCommitter(context);
}

18 View Complete Implementation : DynamicFileListRecordReaderTest.java
Copyright Apache License 2.0
Author : GoogleCloudDataproc
@Before
public void setUp() throws Exception {
    MockitoAnnotations.initMocks(this);
    LoggerConfig.getConfig(DynamicFileListRecordReader.clreplaced).setLevel(Level.FINE);
    // Set up a Configuration which will case "gs://" to grab an InMemoryGoogleHadoopFileSystem.
    config = InMemoryGoogleHadoopFileSystem.getSampleConfiguration();
    when(mockTaskContext.getConfiguration()).thenReturn(config);
    basePath = new Path("gs://foo-bucket/");
    shardPath = new Path(basePath, "shard0/data-*.json");
    estimatedNumRecords = 2;
    fileSystem = basePath.getFileSystem(config);
    fileSystem.mkdirs(shardPath.getParent());
    // Instead of actually blocking, make our mockSleeper throw an exception that we can catch
    // whenever the reader would otherwise be blocking.
    doThrow(new RuntimeException(SLEEP_ID)).when(mockSleeper).sleep(anyLong());
    resetRecordReader();
}

18 View Complete Implementation : GsonRecordReaderTest.java
Copyright Apache License 2.0
Author : GoogleCloudDataproc
/**
 * Helper function to get GsonRecordReader with multiple records.
 *
 * @param splitLength the length of the inputSplit in number of chars.
 * @throws IOException on IO Error.
 */
public GsonRecordReader getRecordReader(int splitLength) throws Exception {
    // Create the task context.
    TaskAttemptContext mockJob = Mockito.mock(TaskAttemptContext.clreplaced);
    Mockito.when(mockJob.getConfiguration()).thenReturn(config);
    Mockito.when(mockJob.getTaskAttemptID()).thenReturn(testTaskAttemptId);
    // Write values to file.
    Path mockPath = new Path("gs://test_bucket/test-object");
    writeFile(ghfs, mockPath, (value1 + "\n" + value2 + "\n").getBytes(UTF_8));
    // Create a new InputSplit containing the values.
    UnshardedInputSplit inputSplit = new UnshardedInputSplit(mockPath, 0, splitLength, new String[0]);
    // Create the GsonRecordReader.
    GsonRecordReader reader = new GsonRecordReader();
    reader.initialize(inputSplit, mockJob);
    return reader;
}

18 View Complete Implementation : ContentIndexingColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Test
public void testHandlerListNormalizedNumericsNoSpace() throws Exception {
    ctx.getConfiguration().set("test" + "." + NUMERIC_LIST + BaseIngestHelper.FIELD_TYPE, NumberType.clreplaced.getName());
    ctx.getConfiguration().set("test" + "." + ContentBaseIngestHelper.LIST_DELIMITERS, LIST_DELIMITERS);
    TypeRegistry.reset();
    TypeRegistry.getInstance(ctx.getConfiguration());
    setupMocks();
    TestContentIndexingColumnBasedHandler handler = new TestContentIndexingColumnBasedHandler();
    handler.setup(ctx);
    helper.setup(ctx.getConfiguration());
    testProcessing(handler, NUMERIC_LIST, LIST_VALUE, listExpectedNumericFields, listExpectedNumericIndex, listExpectedNumericReverse, listExpectedNumericTfValues, false);
}

18 View Complete Implementation : ContentIndexingColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Test
public void testHandlerNormalizedTokenizedFieldNoSpace() throws Exception {
    TypeRegistry.reset();
    TypeRegistry.getInstance(ctx.getConfiguration());
    setupMocks();
    TestContentIndexingColumnBasedHandler handler = new TestContentIndexingColumnBasedHandler();
    handler.setup(ctx);
    helper.setup(ctx.getConfiguration());
    testProcessing(handler, ALPHANUM_LIST, LIST_VALUE, tokenizedExpectedFields, tokenizedExpectedIndex, tokenizedExpectedReverse, tokenizedExpectedTfValues, true);
}

18 View Complete Implementation : AbstractColumnBasedHandler.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Override
public void setup(TaskAttemptContext context) {
    super.setup(context);
    TypeRegistry registry = TypeRegistry.getInstance(context.getConfiguration());
    Type type = registry.get(context.getConfiguration().get(DataTypeHelper.Properties.DATA_NAME));
    helper = type.getIngestHelper(context.getConfiguration());
    log.info(this.getClreplaced().getSimpleName() + " configured.");
}

18 View Complete Implementation : ContentCSVIndexingColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
private CSVRecordReader getCSVRecordReader(String file) throws IOException, URISyntaxException {
    InputSplit split = ColumnBasedHandlerTestUtil.getSplit(file);
    TaskAttemptContext ctx = new TaskAttemptContextImpl(conf, new TaskAttemptID());
    TypeRegistry.reset();
    TypeRegistry.getInstance(ctx.getConfiguration());
    log.debug(TypeRegistry.getContents());
    CSVRecordReader reader = new CSVRecordReader();
    reader.initialize(split, ctx);
    return reader;
}

18 View Complete Implementation : ContentIndexingColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Test
public void testHandlerListNormalizedAlphanumWithEmptyEntry() throws Exception {
    ctx.getConfiguration().set("test" + "." + ALPHANUM_LIST + BaseIngestHelper.FIELD_TYPE, LcNoDiacriticsType.clreplaced.getName());
    ctx.getConfiguration().set("test" + "." + ContentBaseIngestHelper.LIST_DELIMITERS, LIST_DELIMITERS);
    TypeRegistry.reset();
    TypeRegistry.getInstance(ctx.getConfiguration());
    setupMocks();
    TestContentIndexingColumnBasedHandler handler = new TestContentIndexingColumnBasedHandler();
    handler.setup(ctx);
    helper.setup(ctx.getConfiguration());
    testProcessing(handler, ALPHANUM_LIST, LIST_VALUE_WITH_EMPTY_ENTRY, listExpectedAlpahnumFields, listExpectedAlpahnumIndex, listExpectedAlpahnumReverse, listExpectedAlphanumTfValues, false);
}

18 View Complete Implementation : StatsDHelper.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
public void setup(TaskAttemptContext context) {
    setup(context.getConfiguration());
}

18 View Complete Implementation : ContentIndexingColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
@Test
public void testHandlerListNormalizedNumericWithSpace() throws Exception {
    ctx.getConfiguration().set("test" + "." + NUMERIC_LIST + BaseIngestHelper.FIELD_TYPE, NumberType.clreplaced.getName());
    ctx.getConfiguration().set("test" + "." + ContentBaseIngestHelper.LIST_DELIMITERS, LIST_DELIMITERS);
    TypeRegistry.reset();
    TypeRegistry.getInstance(ctx.getConfiguration());
    setupMocks();
    TestContentIndexingColumnBasedHandler handler = new TestContentIndexingColumnBasedHandler();
    handler.setup(ctx);
    helper.setup(ctx.getConfiguration());
    testProcessing(handler, NUMERIC_LIST, LIST_VALUE_WITH_SPACE, listExpectedNumericFields, listExpectedNumericIndex, listExpectedNumericReverse, listExpectedNumericTfValues, false);
}

18 View Complete Implementation : TransformOutputFormat.java
Copyright Apache License 2.0
Author : marklogic
@Override
public RecordWriter<DoreplacedentURI, VALUEOUT> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    fastLoad = Boolean.valueOf(conf.get(OUTPUT_FAST_LOAD));
    Map<String, ContentSource> sourceMap = getSourceMap(fastLoad, context);
    getMimetypesMap();
    // construct the ContentWriter
    return new TransformWriter<VALUEOUT>(conf, sourceMap, fastLoad, am);
}

17 View Complete Implementation : FileOutputCommitter.java
Copyright Apache License 2.0
Author : yncxcw
@Private
public boolean needsTaskCommit(TaskAttemptContext context, Path taskAttemptPath) throws IOException {
    if (hasOutputPath()) {
        if (taskAttemptPath == null) {
            taskAttemptPath = getTaskAttemptPath(context);
        }
        FileSystem fs = taskAttemptPath.getFileSystem(context.getConfiguration());
        return fs.exists(taskAttemptPath);
    }
    return false;
}

17 View Complete Implementation : FileOutputCommitter.java
Copyright Apache License 2.0
Author : aliyun-beta
@Private
public void abortTask(TaskAttemptContext context, Path taskAttemptPath) throws IOException {
    if (hasOutputPath()) {
        context.progress();
        if (taskAttemptPath == null) {
            taskAttemptPath = getTaskAttemptPath(context);
        }
        FileSystem fs = taskAttemptPath.getFileSystem(context.getConfiguration());
        if (!fs.delete(taskAttemptPath, true)) {
            LOG.warn("Could not delete " + taskAttemptPath);
        }
    } else {
        LOG.warn("Output Path is null in abortTask()");
    }
}

17 View Complete Implementation : MapFileOutputFormat.java
Copyright Apache License 2.0
Author : apache
public RecordWriter<WritableComparable<?>, Writable> getRecordWriter(TaskAttemptContext context) throws IOException {
    Configuration conf = context.getConfiguration();
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(context)) {
        // find the kind of compression to do
        compressionType = SequenceFileOutputFormat.getOutputCompressionType(context);
        // find the right codec
        Clreplaced<?> codecClreplaced = getOutputCompressorClreplaced(context, DefaultCodec.clreplaced);
        codec = (CompressionCodec) ReflectionUtils.newInstance(codecClreplaced, conf);
    }
    Path file = getDefaultWorkFile(context, "");
    FileSystem fs = file.getFileSystem(conf);
    // ignore the progress parameter, since MapFile is local
    final MapFile.Writer out = new MapFile.Writer(conf, fs, file.toString(), context.getOutputKeyClreplaced().replacedubclreplaced(WritableComparable.clreplaced), context.getOutputValueClreplaced().replacedubclreplaced(Writable.clreplaced), compressionType, codec, context);
    return new RecordWriter<WritableComparable<?>, Writable>() {

        public void write(WritableComparable<?> key, Writable value) throws IOException {
            out.append(key, value);
        }

        public void close(TaskAttemptContext context) throws IOException {
            out.close();
        }
    };
}

17 View Complete Implementation : CompositeInputFormat.java
Copyright Apache License 2.0
Author : yncxcw
/**
 * Construct a CompositeRecordReader for the children of this InputFormat
 * as defined in the init expression.
 * The outermost join need only be composable, not necessarily a composite.
 * Mandating TupleWritable isn't strictly correct.
 */
// child types unknown
@SuppressWarnings("unchecked")
public RecordReader<K, TupleWritable> createRecordReader(InputSplit split, TaskAttemptContext taskContext) throws IOException, InterruptedException {
    setFormat(taskContext.getConfiguration());
    return root.createRecordReader(split, taskContext);
}

17 View Complete Implementation : FileOutputCommitter.java
Copyright Apache License 2.0
Author : yncxcw
@Private
public void abortTask(TaskAttemptContext context, Path taskAttemptPath) throws IOException {
    if (hasOutputPath()) {
        context.progress();
        if (taskAttemptPath == null) {
            taskAttemptPath = getTaskAttemptPath(context);
        }
        FileSystem fs = taskAttemptPath.getFileSystem(context.getConfiguration());
        if (!fs.delete(taskAttemptPath, true)) {
            LOG.warn("Could not delete " + taskAttemptPath);
        }
    } else {
        LOG.warn("Output Path is null in abortTask()");
    }
}

17 View Complete Implementation : CompositeInputFormat.java
Copyright Apache License 2.0
Author : aliyun-beta
/**
 * Construct a CompositeRecordReader for the children of this InputFormat
 * as defined in the init expression.
 * The outermost join need only be composable, not necessarily a composite.
 * Mandating TupleWritable isn't strictly correct.
 */
// child types unknown
@SuppressWarnings("unchecked")
public RecordReader<K, TupleWritable> createRecordReader(InputSplit split, TaskAttemptContext taskContext) throws IOException, InterruptedException {
    setFormat(taskContext.getConfiguration());
    return root.createRecordReader(split, taskContext);
}

17 View Complete Implementation : MapFileOutputFormat.java
Copyright Apache License 2.0
Author : aliyun-beta
public RecordWriter<WritableComparable<?>, Writable> getRecordWriter(TaskAttemptContext context) throws IOException {
    Configuration conf = context.getConfiguration();
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(context)) {
        // find the kind of compression to do
        compressionType = SequenceFileOutputFormat.getOutputCompressionType(context);
        // find the right codec
        Clreplaced<?> codecClreplaced = getOutputCompressorClreplaced(context, DefaultCodec.clreplaced);
        codec = (CompressionCodec) ReflectionUtils.newInstance(codecClreplaced, conf);
    }
    Path file = getDefaultWorkFile(context, "");
    FileSystem fs = file.getFileSystem(conf);
    // ignore the progress parameter, since MapFile is local
    final MapFile.Writer out = new MapFile.Writer(conf, fs, file.toString(), context.getOutputKeyClreplaced().replacedubclreplaced(WritableComparable.clreplaced), context.getOutputValueClreplaced().replacedubclreplaced(Writable.clreplaced), compressionType, codec, context);
    return new RecordWriter<WritableComparable<?>, Writable>() {

        public void write(WritableComparable<?> key, Writable value) throws IOException {
            out.append(key, value);
        }

        public void close(TaskAttemptContext context) throws IOException {
            out.close();
        }
    };
}

17 View Complete Implementation : FileOutputCommitter.java
Copyright Apache License 2.0
Author : aliyun-beta
@Private
public boolean needsTaskCommit(TaskAttemptContext context, Path taskAttemptPath) throws IOException {
    if (hasOutputPath()) {
        if (taskAttemptPath == null) {
            taskAttemptPath = getTaskAttemptPath(context);
        }
        FileSystem fs = taskAttemptPath.getFileSystem(context.getConfiguration());
        return fs.exists(taskAttemptPath);
    }
    return false;
}

17 View Complete Implementation : FixedLengthRecordReader.java
Copyright Apache License 2.0
Author : yncxcw
@Override
public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
    FileSplit split = (FileSplit) genericSplit;
    Configuration job = context.getConfiguration();
    final Path file = split.getPath();
    initialize(job, split.getStart(), split.getLength(), file);
}

17 View Complete Implementation : ForwardingBigQueryFileOutputFormatTest.java
Copyright Apache License 2.0
Author : GoogleCloudDataproc
/**
 * Verify exceptions are being thrown.
 */
/**
 * Sets up common objects for testing before each test.
 */
@Before
public void setUp() throws Exception {
    // Generate Mocks.
    MockitoAnnotations.initMocks(this);
    // Create the file system.
    ghfs = new InMemoryGoogleHadoopFileSystem();
    // Create the configuration, but setup in the tests.
    job = Job.getInstance(InMemoryGoogleHadoopFileSystem.getSampleConfiguration());
    conf = job.getConfiguration();
    CredentialConfigurationUtil.addTestConfigurationSettings(conf);
    BigQueryOutputConfiguration.configureWithAutoSchema(conf, QUALIFIED_TEST_TABLE_ID, TEST_OUTPUT_PATH_STRING, TEST_FILE_FORMAT, TEST_OUTPUT_CLreplaced);
    // Configure mocks.
    when(mockTaskAttemptContext.getConfiguration()).thenReturn(conf);
    when(mockTaskAttemptContext.getTaskAttemptID()).thenReturn(TEST_TASK_ATTEMPT_ID);
    when(mockFileOutputFormat.getOutputCommitter(eq(mockTaskAttemptContext))).thenReturn(mockOutputCommitter);
    when(mockFileOutputFormat.getRecordWriter(eq(mockTaskAttemptContext))).thenReturn(mockRecordWriter);
    // Create and setup the output format.
    outputFormat = new ForwardingBigQueryFileOutputFormat<Text, Text>();
    outputFormat.setDelegate(mockFileOutputFormat);
}

17 View Complete Implementation : PathOutputCommitterFactory.java
Copyright Apache License 2.0
Author : apache
/**
 * Create the committer factory for a task attempt and destination, then
 * create the committer from it.
 * @param outputPath the task's output path, or or null if no output path
 * has been defined.
 * @param context the task attempt context
 * @return the committer to use
 * @throws IOException problems instantiating the committer
 */
public static PathOutputCommitter createCommitter(Path outputPath, TaskAttemptContext context) throws IOException {
    return getCommitterFactory(outputPath, context.getConfiguration()).createOutputCommitter(outputPath, context);
}

17 View Complete Implementation : LazyOutputFormat.java
Copyright Apache License 2.0
Author : apache
@Override
public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException {
    if (baseOut == null) {
        getBaseOutputFormat(context.getConfiguration());
    }
    return super.getOutputCommitter(context);
}

17 View Complete Implementation : FederatedBigQueryOutputCommitterTest.java
Copyright Apache License 2.0
Author : GoogleCloudDataproc
// Verify exceptions are being thrown.
// Sets up common objects for testing before each test.
@Before
public void setUp() throws IOException {
    // Generate Mocks.
    MockitoAnnotations.initMocks(this);
    // Create the file system.
    ghfs = new InMemoryGoogleHadoopFileSystem();
    // Setup the configuration.
    job = Job.getInstance(InMemoryGoogleHadoopFileSystem.getSampleConfiguration());
    conf = job.getConfiguration();
    CredentialConfigurationUtil.addTestConfigurationSettings(conf);
    BigQueryOutputConfiguration.configure(conf, QUALIFIED_TEST_TABLE_ID, TEST_TABLE_SCHEMA, TEST_OUTPUT_PATH_STRING, TEST_FILE_FORMAT, TEST_OUTPUT_CLreplaced);
    // Setup sample data.
    outputTableRef = BigQueryOutputConfiguration.getTableReference(conf);
    outputPath = BigQueryOutputConfiguration.getGcsOutputPath(conf);
    outputSampleFilePath = new Path(TEST_OUTPUT_FILE_STRING);
    // Configure mocks.
    when(mockTaskAttemptContext.getConfiguration()).thenReturn(conf);
    when(mockTaskAttemptContext.getTaskAttemptID()).thenReturn(TEST_TASK_ATTEMPT_ID);
    // Setup committer.
    committer = new FederatedBigQueryOutputCommitter(mockTaskAttemptContext, mockCommitter);
    committer.setBigQueryHelper(mockBigQueryHelper);
}

17 View Complete Implementation : DBInputFormat.java
Copyright Apache License 2.0
Author : aliyun-beta
/**
 * {@inheritDoc}
 */
public RecordReader<LongWritable, T> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    return createDBRecordReader((DBInputSplit) split, context.getConfiguration());
}

17 View Complete Implementation : CompositeInputFormat.java
Copyright Apache License 2.0
Author : apache
/**
 * Construct a CompositeRecordReader for the children of this InputFormat
 * as defined in the init expression.
 * The outermost join need only be composable, not necessarily a composite.
 * Mandating TupleWritable isn't strictly correct.
 */
// child types unknown
@SuppressWarnings("unchecked")
public RecordReader<K, TupleWritable> createRecordReader(InputSplit split, TaskAttemptContext taskContext) throws IOException, InterruptedException {
    setFormat(taskContext.getConfiguration());
    return root.createRecordReader(split, taskContext);
}

17 View Complete Implementation : DBInputFormat.java
Copyright Apache License 2.0
Author : apache
/**
 * {@inheritDoc}
 */
public RecordReader<LongWritable, T> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    return createDBRecordReader((DBInputSplit) split, context.getConfiguration());
}

17 View Complete Implementation : NYCTLCColumnBasedHandlerTest.java
Copyright Apache License 2.0
Author : NationalSecurityAgency
private NYCTLCReader getNYCTLCRecordReader(String file) throws IOException, URISyntaxException {
    InputSplit split = ColumnBasedHandlerTestUtil.getSplit(file);
    TaskAttemptContext ctx = new TaskAttemptContextImpl(conf, new TaskAttemptID());
    TypeRegistry.reset();
    TypeRegistry.getInstance(ctx.getConfiguration());
    log.debug(TypeRegistry.getContents());
    NYCTLCReader reader = new NYCTLCReader();
    reader.initialize(split, ctx);
    return reader;
}

17 View Complete Implementation : FixedLengthRecordReader.java
Copyright Apache License 2.0
Author : apache
@Override
public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
    FileSplit split = (FileSplit) genericSplit;
    Configuration job = context.getConfiguration();
    final Path file = split.getPath();
    initialize(job, split.getStart(), split.getLength(), file);
}

17 View Complete Implementation : FileOutputCommitter.java
Copyright Apache License 2.0
Author : apache
@Private
public boolean needsTaskCommit(TaskAttemptContext context, Path taskAttemptPath) throws IOException {
    if (hasOutputPath()) {
        if (taskAttemptPath == null) {
            taskAttemptPath = getTaskAttemptPath(context);
        }
        FileSystem fs = taskAttemptPath.getFileSystem(context.getConfiguration());
        return fs.exists(taskAttemptPath);
    }
    return false;
}

17 View Complete Implementation : FileOutputCommitter.java
Copyright Apache License 2.0
Author : apache
@Private
public void abortTask(TaskAttemptContext context, Path taskAttemptPath) throws IOException {
    if (hasOutputPath()) {
        context.progress();
        if (taskAttemptPath == null) {
            taskAttemptPath = getTaskAttemptPath(context);
        }
        FileSystem fs = taskAttemptPath.getFileSystem(context.getConfiguration());
        if (!fs.delete(taskAttemptPath, true)) {
            LOG.warn("Could not delete " + taskAttemptPath);
        }
    } else {
        LOG.warn("Output Path is null in abortTask()");
    }
}

17 View Complete Implementation : LazyOutputFormat.java
Copyright Apache License 2.0
Author : apache
@Override
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
    if (baseOut == null) {
        getBaseOutputFormat(context.getConfiguration());
    }
    return new LazyRecordWriter<K, V>(baseOut, context);
}