org.apache.hadoop.mapreduce.TaskAttemptContext - java examples

Here are the examples of the java api org.apache.hadoop.mapreduce.TaskAttemptContext taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

155 Examples 7

19 View Complete Implementation : StagingCommitter.java
Copyright Apache License 2.0
Author : apache
/**
 * Compute the path where the output of a committed task is stored until
 * the entire job is committed.
 * @param context the context of the task attempt
 * @return the path where the output of a committed task is stored until
 * the entire job is committed.
 */
public Path getCommittedTaskPath(TaskAttemptContext context) {
    return getCommittedTaskPath(getAppAttemptId(context), context);
}

19 View Complete Implementation : PathOutputCommitterFactory.java
Copyright Apache License 2.0
Author : apache
/**
 * Create an instance of the default committer, a {@link FileOutputCommitter}
 * for a task.
 * @param outputPath the task's output path, or or null if no output path
 * has been defined.
 * @param context the task attempt context
 * @return the committer to use
 * @throws IOException problems instantiating the committer
 */
protected final PathOutputCommitter createFileOutputCommitter(Path outputPath, TaskAttemptContext context) throws IOException {
    LOG.debug("Creating FileOutputCommitter for path {} and context {}", outputPath, context);
    return new FileOutputCommitter(outputPath, context);
}

19 View Complete Implementation : SequenceFileAsBinaryOutputFormat.java
Copyright Apache License 2.0
Author : apache
@Override
public RecordWriter<BytesWritable, BytesWritable> getRecordWriter(TaskAttemptContext context) throws IOException {
    final SequenceFile.Writer out = getSequenceWriter(context, getSequenceFileOutputKeyClreplaced(context), getSequenceFileOutputValueClreplaced(context));
    return new RecordWriter<BytesWritable, BytesWritable>() {

        private WritableValueBytes wvaluebytes = new WritableValueBytes();

        public void write(BytesWritable bkey, BytesWritable bvalue) throws IOException {
            wvaluebytes.reset(bvalue);
            out.appendRaw(bkey.getBytes(), 0, bkey.getLength(), wvaluebytes);
            wvaluebytes.reset(null);
        }

        public void close(TaskAttemptContext context) throws IOException {
            out.close();
        }
    };
}

19 View Complete Implementation : AbstractS3ACommitter.java
Copyright Apache License 2.0
Author : apache
/**
 * Get the task attempt path filesystem. This may not be the same as the
 * final destination FS, and so may not be an S3A FS.
 * @param context task attempt
 * @return the filesystem
 * @throws IOException failure to instantiate
 */
protected FileSystem getTaskAttemptFilesystem(TaskAttemptContext context) throws IOException {
    return getTaskAttemptPath(context).getFileSystem(getConf());
}

19 View Complete Implementation : FileOutputFormat.java
Copyright Apache License 2.0
Author : apache
/**
 * Generate a unique filename, based on the task id, name, and extension
 * @param context the task that is calling this
 * @param name the base filename
 * @param extension the filename extension
 * @return a string like $name-[mrsct]-$id$extension
 */
public synchronized static String getUniqueFile(TaskAttemptContext context, String name, String extension) {
    TaskID taskId = context.getTaskAttemptID().getTaskID();
    int parreplacedion = taskId.getId();
    StringBuilder result = new StringBuilder();
    result.append(name);
    result.append('-');
    result.append(TaskID.getRepresentingCharacter(taskId.getTaskType()));
    result.append('-');
    result.append(NUMBER_FORMAT.format(parreplacedion));
    result.append(extension);
    return result.toString();
}

19 View Complete Implementation : LoggingTextOutputFormat.java
Copyright Apache License 2.0
Author : apache
@Override
public LoggingLineRecordWriter<K, V> getRecordWriter(TaskAttemptContext job) throws IOException, InterruptedException {
    Configuration conf = job.getConfiguration();
    boolean isCompressed = getCompressOutput(job);
    String keyValueSeparator = conf.get(SEPARATOR, "\t");
    CompressionCodec codec = null;
    String extension = "";
    if (isCompressed) {
        Clreplaced<? extends CompressionCodec> codecClreplaced = getOutputCompressorClreplaced(job, GzipCodec.clreplaced);
        codec = ReflectionUtils.newInstance(codecClreplaced, conf);
        extension = codec.getDefaultExtension();
    }
    Path file = getDefaultWorkFile(job, extension);
    FileSystem fs = file.getFileSystem(conf);
    FSDataOutputStream fileOut = fs.create(file, false);
    LOG.debug("Creating LineRecordWriter with destination {}", file);
    if (isCompressed) {
        return new LoggingLineRecordWriter<>(file, new DataOutputStream(codec.createOutputStream(fileOut)), keyValueSeparator);
    } else {
        return new LoggingLineRecordWriter<>(file, fileOut, keyValueSeparator);
    }
}

19 View Complete Implementation : ITestMagicCommitProtocol.java
Copyright Apache License 2.0
Author : apache
/**
 * The magic committer paths are always on S3, and always have
 * "__magic" in the path.
 * @param committer committer instance
 * @param context task attempt context
 * @throws IOException IO failure
 */
@Override
protected void validateTaskAttemptWorkingDirectory(final AbstractS3ACommitter committer, final TaskAttemptContext context) throws IOException {
    URI wd = committer.getWorkPath().toUri();
    replacedertEquals("Wrong schema for working dir " + wd + " with committer " + committer, "s3a", wd.getScheme());
    replacedertThat(wd.getPath(), containsString('/' + CommitConstants.MAGIC + '/'));
}

19 View Complete Implementation : NamedCommitterFactory.java
Copyright Apache License 2.0
Author : apache
@SuppressWarnings("JavaReflectionMemberAccess")
@Override
public PathOutputCommitter createOutputCommitter(Path outputPath, TaskAttemptContext context) throws IOException {
    Clreplaced<? extends PathOutputCommitter> clazz = loadCommitterClreplaced(context);
    LOG.debug("Using PathOutputCommitter implementation {}", clazz);
    try {
        Constructor<? extends PathOutputCommitter> ctor = clazz.getConstructor(Path.clreplaced, TaskAttemptContext.clreplaced);
        return ctor.newInstance(outputPath, context);
    } catch (NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) {
        throw new IOException("Failed to create " + clazz + ":" + e, e);
    }
}

19 View Complete Implementation : BindingPathOutputCommitter.java
Copyright Apache License 2.0
Author : apache
@Override
public void recoverTask(TaskAttemptContext taskContext) throws IOException {
    committer.recoverTask(taskContext);
}

19 View Complete Implementation : AbstractITCommitProtocol.java
Copyright Apache License 2.0
Author : apache
/**
 * Perform any actions needed to validate the working directory of
 * a committer.
 * For example: filesystem, path attributes
 * @param committer committer instance
 * @param context task attempt context
 * @throws IOException IO failure
 */
protected void validateTaskAttemptWorkingDirectory(AbstractS3ACommitter committer, TaskAttemptContext context) throws IOException {
}

19 View Complete Implementation : FileOutputFormat.java
Copyright Apache License 2.0
Author : apache
/**
 * Get the default path and filename for the output format.
 * @param context the task context
 * @param extension an extension to add to the filename
 * @return a full path $output/_temporary/$taskid/part-[mr]-$id
 * @throws IOException
 */
public Path getDefaultWorkFile(TaskAttemptContext context, String extension) throws IOException {
    OutputCommitter c = getOutputCommitter(context);
    Preconditions.checkState(c instanceof PathOutputCommitter, "Committer %s is not a PathOutputCommitter", c);
    Path workPath = ((PathOutputCommitter) c).getWorkPath();
    Preconditions.checkNotNull(workPath, "Null workPath returned by committer %s", c);
    Path workFile = new Path(workPath, getUniqueFile(context, getOutputName(context), extension));
    LOG.debug("Work file for {} extension '{}' is {}", context, extension, workFile);
    return workFile;
}

19 View Complete Implementation : SequenceFileOutputFormat.java
Copyright Apache License 2.0
Author : apache
protected SequenceFile.Writer getSequenceWriter(TaskAttemptContext context, Clreplaced<?> keyClreplaced, Clreplaced<?> valueClreplaced) throws IOException {
    Configuration conf = context.getConfiguration();
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(context)) {
        // find the kind of compression to do
        compressionType = getOutputCompressionType(context);
        // find the right codec
        Clreplaced<?> codecClreplaced = getOutputCompressorClreplaced(context, DefaultCodec.clreplaced);
        codec = (CompressionCodec) ReflectionUtils.newInstance(codecClreplaced, conf);
    }
    // get the path of the temporary output file
    Path file = getDefaultWorkFile(context, "");
    FileSystem fs = file.getFileSystem(conf);
    return SequenceFile.createWriter(fs, conf, file, keyClreplaced, valueClreplaced, compressionType, codec, context);
}

19 View Complete Implementation : TestPathOutputCommitterFactory.java
Copyright Apache License 2.0
Author : apache
/**
 * Create a committer from a task context, via
 * {@link PathOutputCommitterFactory#createCommitter(Path, TaskAttemptContext)}.
 * @param <U> type of committer
 * @param committerClreplaced expected committer clreplaced
 * @param path output path (may be null)
 * @param context task attempt context
 * @return the committer
 * @throws IOException failure to create
 */
private <U extends PathOutputCommitter> U createCommitter(Clreplaced<U> committerClreplaced, Path path, TaskAttemptContext context) throws IOException {
    PathOutputCommitter committer = PathOutputCommitterFactory.createCommitter(path, context);
    replacedertEquals(" Wrong committer for path " + path, committerClreplaced, committer.getClreplaced());
    return (U) committer;
}

19 View Complete Implementation : TestMRCJCFileOutputCommitter.java
Copyright Apache License 2.0
Author : apache
@Test
public void testEmptyOutput() throws Exception {
    Job job = Job.getInstance();
    FileOutputFormat.setOutputPath(job, outDir);
    Configuration conf = job.getConfiguration();
    conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
    JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
    TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
    FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);
    // setup
    committer.setupJob(jContext);
    committer.setupTask(tContext);
    // Do not write any output
    // do commit
    committer.commitTask(tContext);
    committer.commitJob(jContext);
    FileUtil.fullyDelete(new File(outDir.toString()));
}

19 View Complete Implementation : FileOutputCommitter.java
Copyright Apache License 2.0
Author : apache
/**
 * Compute the path where the output of a committed task is stored until
 * the entire job is committed.
 * @param context the context of the task attempt
 * @return the path where the output of a committed task is stored until
 * the entire job is committed.
 */
public Path getCommittedTaskPath(TaskAttemptContext context) {
    return getCommittedTaskPath(getAppAttemptId(context), context);
}

19 View Complete Implementation : CommitUtilsWithMR.java
Copyright Apache License 2.0
Author : apache
/**
 * Compute the path where the output of a task attempt is stored until
 * that task is committed.
 * This path is marked as a base path for relocations, so subdirectory
 * information is preserved.
 * @param context the context of the task attempt.
 * @param dest The output path to commit work into
 * @return the path where a task attempt should be stored.
 */
public static Path getMagicTaskAttemptPath(TaskAttemptContext context, Path dest) {
    return new Path(getBaseMagicTaskAttemptPath(context, dest), BASE);
}

19 View Complete Implementation : AbstractITCommitProtocol.java
Copyright Apache License 2.0
Author : apache
/**
 * Commit a task then validate the state of the committer afterwards.
 * @param committer committer
 * @param tContext task context
 * @throws IOException IO failure
 */
protected void commitTask(final AbstractS3ACommitter committer, final TaskAttemptContext tContext) throws IOException {
    committer.commitTask(tContext);
    verifyCommitterHasNoThreads(committer);
}

19 View Complete Implementation : TestMRCJCFileOutputCommitter.java
Copyright Apache License 2.0
Author : apache
@SuppressWarnings("unchecked")
@Test
public void testAbort() throws IOException, InterruptedException {
    Job job = Job.getInstance();
    FileOutputFormat.setOutputPath(job, outDir);
    Configuration conf = job.getConfiguration();
    conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
    JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
    TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
    FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);
    // do setup
    committer.setupJob(jContext);
    committer.setupTask(tContext);
    // write output
    TextOutputFormat theOutputFormat = new TextOutputFormat();
    RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(tContext);
    writeOutput(theRecordWriter, tContext);
    // do abort
    committer.abortTask(tContext);
    File expectedFile = new File(new Path(committer.getWorkPath(), partFile).toString());
    replacedertFalse("task temp dir still exists", expectedFile.exists());
    committer.abortJob(jContext, JobStatus.State.FAILED);
    expectedFile = new File(new Path(outDir, FileOutputCommitter.PENDING_DIR_NAME).toString());
    replacedertFalse("job temp dir still exists", expectedFile.exists());
    replacedertThat(new File(outDir.toString()).listFiles()).withFailMessage("Output directory not empty").isEmpty();
    FileUtil.fullyDelete(new File(outDir.toString()));
}

19 View Complete Implementation : TestMRCJCFileOutputCommitter.java
Copyright Apache License 2.0
Author : apache
@SuppressWarnings("unchecked")
private void writeOutput(RecordWriter theRecordWriter, TaskAttemptContext context) throws IOException, InterruptedException {
    NullWritable nullWritable = NullWritable.get();
    try {
        theRecordWriter.write(key1, val1);
        theRecordWriter.write(null, nullWritable);
        theRecordWriter.write(null, val1);
        theRecordWriter.write(nullWritable, val2);
        theRecordWriter.write(key2, nullWritable);
        theRecordWriter.write(key1, null);
        theRecordWriter.write(null, null);
        theRecordWriter.write(key2, val2);
    } finally {
        theRecordWriter.close(context);
    }
}

19 View Complete Implementation : PathOutputCommitterFactory.java
Copyright Apache License 2.0
Author : apache
/**
 * Create an output committer for a task attempt.
 * @param outputPath output path. This may be null.
 * @param context context
 * @return a new committer
 * @throws IOException problems instantiating the committer
 */
public PathOutputCommitter createOutputCommitter(Path outputPath, TaskAttemptContext context) throws IOException {
    return createFileOutputCommitter(outputPath, context);
}

19 View Complete Implementation : CombineFileRecordReaderWrapper.java
Copyright Apache License 2.0
Author : apache
private boolean fileSplitIsValid(TaskAttemptContext context) {
    Configuration conf = context.getConfiguration();
    long offset = conf.getLong(MRJobConfig.MAP_INPUT_START, 0L);
    if (fileSplit.getStart() != offset) {
        return false;
    }
    long length = conf.getLong(MRJobConfig.MAP_INPUT_PATH, 0L);
    if (fileSplit.getLength() != length) {
        return false;
    }
    String path = conf.get(MRJobConfig.MAP_INPUT_FILE);
    if (!fileSplit.getPath().toString().equals(path)) {
        return false;
    }
    return true;
}

19 View Complete Implementation : CompositeInputFormat.java
Copyright Apache License 2.0
Author : apache
/**
 * Construct a CompositeRecordReader for the children of this InputFormat
 * as defined in the init expression.
 * The outermost join need only be composable, not necessarily a composite.
 * Mandating TupleWritable isn't strictly correct.
 */
// child types unknown
@SuppressWarnings("unchecked")
public RecordReader<K, TupleWritable> createRecordReader(InputSplit split, TaskAttemptContext taskContext) throws IOException, InterruptedException {
    setFormat(taskContext.getConfiguration());
    return root.createRecordReader(split, taskContext);
}

19 View Complete Implementation : WrappedRecordReader.java
Copyright Apache License 2.0
Author : apache
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    rr.initialize(split, context);
    conf = context.getConfiguration();
    nextKeyValue();
    if (!empty) {
        keyclreplaced = key.getClreplaced().replacedubclreplaced(WritableComparable.clreplaced);
        valueclreplaced = value.getClreplaced();
        if (cmp == null) {
            cmp = WritableComparator.get(keyclreplaced, conf);
        }
    }
}

19 View Complete Implementation : PartitionedStagingCommitter.java
Copyright Apache License 2.0
Author : apache
@Override
protected int commitTaskInternal(TaskAttemptContext context, List<? extends FileStatus> taskOutput) throws IOException {
    Path attemptPath = getTaskAttemptPath(context);
    Set<String> parreplacedions = Paths.getParreplacedions(attemptPath, taskOutput);
    // enforce conflict resolution, but only if the mode is FAIL. for APPEND,
    // it doesn't matter that the parreplacedions are already there, and for REPLACE,
    // deletion should be done during job commit.
    FileSystem fs = getDestFS();
    if (getConflictResolutionMode(context, fs.getConf()) == ConflictResolution.FAIL) {
        for (String parreplacedion : parreplacedions) {
            // getFinalPath adds the UUID to the file name. this needs the parent.
            Path parreplacedionPath = getFinalPath(parreplacedion + "/file", context).getParent();
            if (fs.exists(parreplacedionPath)) {
                throw failDestinationExists(parreplacedionPath, "Committing task " + context.getTaskAttemptID());
            }
        }
    }
    return super.commitTaskInternal(context, taskOutput);
}

19 View Complete Implementation : SequenceFileInputFilter.java
Copyright Apache License 2.0
Author : apache
/**
 * Create a record reader for the given split
 * @param split file split
 * @param context the task-attempt context
 * @return RecordReader
 */
public RecordReader<K, V> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException {
    context.setStatus(split.toString());
    return new FilterRecordReader<K, V>(context.getConfiguration());
}

19 View Complete Implementation : FileOutputCommitter.java
Copyright Apache License 2.0
Author : apache
/**
 * Compute the path where the output of a task attempt is stored until
 * that task is committed.
 *
 * @param context the context of the task attempt.
 * @return the path where a task attempt should be stored.
 */
public Path getTaskAttemptPath(TaskAttemptContext context) {
    return new Path(getPendingTaskAttemptsPath(context), String.valueOf(context.getTaskAttemptID()));
}

19 View Complete Implementation : DynamicInputChunkContext.java
Copyright Apache License 2.0
Author : apache
public DynamicInputChunk acquire(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
    String taskId = taskAttemptContext.getTaskAttemptID().getTaskID().toString();
    Path acquiredFilePath = new Path(getChunkRootPath(), taskId);
    if (fs.exists(acquiredFilePath)) {
        LOG.info("Acquiring pre-replacedigned chunk: " + acquiredFilePath);
        return new DynamicInputChunk(acquiredFilePath, taskAttemptContext, this);
    }
    for (FileStatus chunkFile : getListOfChunkFiles()) {
        if (fs.rename(chunkFile.getPath(), acquiredFilePath)) {
            LOG.info(taskId + " acquired " + chunkFile.getPath());
            return new DynamicInputChunk(acquiredFilePath, taskAttemptContext, this);
        }
    }
    return null;
}

19 View Complete Implementation : FileOutputCommitter.java
Copyright Apache License 2.0
Author : apache
/**
 * Compute the path where the output of a committed task is stored until the
 * entire job is committed for a specific application attempt.
 * @param appAttemptId the id of the application attempt to use
 * @param context the context of any task.
 * @return the path where the output of a committed task is stored.
 */
protected Path getCommittedTaskPath(int appAttemptId, TaskAttemptContext context) {
    return new Path(getJobAttemptPath(appAttemptId), String.valueOf(context.getTaskAttemptID().getTaskID()));
}

19 View Complete Implementation : TestPreemptableFileOutputCommitter.java
Copyright Apache License 2.0
Author : apache
@Test
public void testPartialOutputCleanup() throws FileNotFoundException, IllegalArgumentException, IOException {
    Configuration conf = new Configuration(false);
    conf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 1);
    TaskAttemptID tid0 = new TaskAttemptID("1363718006656", 1, TaskType.REDUCE, 14, 3);
    Path p = spy(new Path("/user/hadoop/out"));
    Path a = new Path("hdfs://user/hadoop/out");
    Path p0 = new Path(a, "_temporary/1/attempt_1363718006656_0001_r_000014_0");
    Path p1 = new Path(a, "_temporary/1/attempt_1363718006656_0001_r_000014_1");
    Path p2 = new Path(a, "_temporary/1/attempt_1363718006656_0001_r_000013_0");
    // (p3 does not exist)
    Path p3 = new Path(a, "_temporary/1/attempt_1363718006656_0001_r_000014_2");
    FileStatus[] fsa = new FileStatus[3];
    fsa[0] = new FileStatus();
    fsa[0].setPath(p0);
    fsa[1] = new FileStatus();
    fsa[1].setPath(p1);
    fsa[2] = new FileStatus();
    fsa[2].setPath(p2);
    final FileSystem fs = mock(FileSystem.clreplaced);
    when(fs.exists(eq(p0))).thenReturn(true);
    when(fs.exists(eq(p1))).thenReturn(true);
    when(fs.exists(eq(p2))).thenReturn(true);
    when(fs.exists(eq(p3))).thenReturn(false);
    when(fs.delete(eq(p0), eq(true))).thenReturn(true);
    when(fs.delete(eq(p1), eq(true))).thenReturn(true);
    doReturn(fs).when(p).getFileSystem(any(Configuration.clreplaced));
    when(fs.makeQualified(eq(p))).thenReturn(a);
    TaskAttemptContext context = mock(TaskAttemptContext.clreplaced);
    when(context.getTaskAttemptID()).thenReturn(tid0);
    when(context.getConfiguration()).thenReturn(conf);
    PartialFileOutputCommitter foc = new TestPFOC(p, context, fs);
    foc.cleanUpPartialOutputForTask(context);
    verify(fs).delete(eq(p0), eq(true));
    verify(fs).delete(eq(p1), eq(true));
    verify(fs, times(1)).delete(eq(p3), eq(true));
    verify(fs, never()).delete(eq(p2), eq(true));
}

19 View Complete Implementation : NullOutputFormat.java
Copyright Apache License 2.0
Author : apache
@Override
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) {
    return new RecordWriter<K, V>() {

        public void write(K key, V value) {
        }

        public void close(TaskAttemptContext context) {
        }
    };
}

19 View Complete Implementation : SequenceFileOutputFormat.java
Copyright Apache License 2.0
Author : apache
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
    final SequenceFile.Writer out = getSequenceWriter(context, context.getOutputKeyClreplaced(), context.getOutputValueClreplaced());
    return new RecordWriter<K, V>() {

        public void write(K key, V value) throws IOException {
            out.append(key, value);
        }

        public void close(TaskAttemptContext context) throws IOException {
            out.close();
        }
    };
}

19 View Complete Implementation : UniformSizeInputFormat.java
Copyright Apache License 2.0
Author : apache
/**
 * Implementation of InputFormat::createRecordReader().
 * @param split The split for which the RecordReader is sought.
 * @param context The context of the current task-attempt.
 * @return A SequenceFileRecordReader instance, (since the copy-listing is a
 * simple sequence-file.)
 * @throws IOException
 * @throws InterruptedException
 */
@Override
public RecordReader<Text, CopyListingFileStatus> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    return new SequenceFileRecordReader<Text, CopyListingFileStatus>();
}

19 View Complete Implementation : StagingCommitter.java
Copyright Apache License 2.0
Author : apache
/**
 * Compute the path where the output of a committed task is stored until the
 * entire job is committed for a specific application attempt.
 * @param appAttemptId the ID of the application attempt to use
 * @param context the context of any task.
 * @return the path where the output of a committed task is stored.
 */
protected Path getCommittedTaskPath(int appAttemptId, TaskAttemptContext context) {
    validateContext(context);
    return new Path(getJobAttemptPath(appAttemptId), String.valueOf(context.getTaskAttemptID().getTaskID()));
}

19 View Complete Implementation : DBOutputFormat.java
Copyright Apache License 2.0
Author : apache
/**
 * {@inheritDoc}
 */
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException {
    DBConfiguration dbConf = new DBConfiguration(context.getConfiguration());
    String tableName = dbConf.getOutputTableName();
    String[] fieldNames = dbConf.getOutputFieldNames();
    if (fieldNames == null) {
        fieldNames = new String[dbConf.getOutputFieldCount()];
    }
    try {
        Connection connection = dbConf.getConnection();
        PreparedStatement statement = null;
        DatabaseMetaData dbMeta = connection.getMetaData();
        this.dbProductName = dbMeta.getDatabaseProductName().toUpperCase();
        statement = connection.prepareStatement(constructQuery(tableName, fieldNames));
        return new DBRecordWriter(connection, statement);
    } catch (Exception ex) {
        throw new IOException(ex.getMessage());
    }
}

19 View Complete Implementation : NullOutputFormat.java
Copyright Apache License 2.0
Author : apache
@Override
public OutputCommitter getOutputCommitter(TaskAttemptContext context) {
    return new OutputCommitter() {

        public void abortTask(TaskAttemptContext taskContext) {
        }

        public void cleanupJob(JobContext jobContext) {
        }

        public void commitTask(TaskAttemptContext taskContext) {
        }

        public boolean needsTaskCommit(TaskAttemptContext taskContext) {
            return false;
        }

        public void setupJob(JobContext jobContext) {
        }

        public void setupTask(TaskAttemptContext taskContext) {
        }

        @Override
        @Deprecated
        public boolean isRecoverySupported() {
            return true;
        }

        @Override
        public void recoverTask(TaskAttemptContext taskContext) throws IOException {
        // Nothing to do for recovering the task.
        }
    };
}

19 View Complete Implementation : CommitUtilsWithMR.java
Copyright Apache License 2.0
Author : apache
/**
 * Get the base Magic attempt path, without any annotations to mark relative
 * references.
 * @param context task context.
 * @param dest The output path to commit work into
 * @return the path under which all attempts go
 */
public static Path getBaseMagicTaskAttemptPath(TaskAttemptContext context, Path dest) {
    return new Path(getMagicTaskAttemptsPath(context, dest), String.valueOf(context.getTaskAttemptID()));
}

19 View Complete Implementation : SequenceFileRecordReader.java
Copyright Apache License 2.0
Author : apache
@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    FileSplit fileSplit = (FileSplit) split;
    conf = context.getConfiguration();
    Path path = fileSplit.getPath();
    FileSystem fs = path.getFileSystem(conf);
    this.in = new SequenceFile.Reader(fs, path, conf);
    this.end = fileSplit.getStart() + fileSplit.getLength();
    if (fileSplit.getStart() > in.getPosition()) {
        // sync to start
        in.sync(fileSplit.getStart());
    }
    this.start = in.getPosition();
    more = start < end;
}

19 View Complete Implementation : TestMRCJCFileOutputCommitter.java
Copyright Apache License 2.0
Author : apache
@SuppressWarnings("unchecked")
@Test
public void testCommitter() throws Exception {
    Job job = Job.getInstance();
    FileOutputFormat.setOutputPath(job, outDir);
    Configuration conf = job.getConfiguration();
    conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
    JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
    TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
    FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);
    // setup
    committer.setupJob(jContext);
    committer.setupTask(tContext);
    // write output
    TextOutputFormat theOutputFormat = new TextOutputFormat();
    RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(tContext);
    writeOutput(theRecordWriter, tContext);
    // do commit
    committer.commitTask(tContext);
    committer.commitJob(jContext);
    // validate output
    File expectedFile = new File(new Path(outDir, partFile).toString());
    StringBuffer expectedOutput = new StringBuffer();
    expectedOutput.append(key1).append('\t').append(val1).append("\n");
    expectedOutput.append(val1).append("\n");
    expectedOutput.append(val2).append("\n");
    expectedOutput.append(key2).append("\n");
    expectedOutput.append(key1).append("\n");
    expectedOutput.append(key2).append('\t').append(val2).append("\n");
    String output = UtilsForTests.slurp(expectedFile);
    replacedertThat(output).isEqualTo(expectedOutput.toString());
    FileUtil.fullyDelete(new File(outDir.toString()));
}

19 View Complete Implementation : PartialFileOutputCommitter.java
Copyright Apache License 2.0
Author : apache
@Override
public void cleanUpPartialOutputForTask(TaskAttemptContext context) throws IOException {
    // we double check this is never invoked from a non-preemptable subclreplaced.
    // This should never happen, since the invoking codes is checking it too,
    // but it is safer to double check. Errors handling this would produce
    // inconsistent output.
    if (!this.getClreplaced().isAnnotationPresent(Checkpointable.clreplaced)) {
        throw new IllegalStateException("Invoking cleanUpPartialOutputForTask() " + "from non @Preemptable clreplaced");
    }
    FileSystem fs = fsFor(getTaskAttemptPath(context), context.getConfiguration());
    LOG.info("cleanUpPartialOutputForTask: removing everything belonging to " + context.getTaskAttemptID().getTaskID() + " in: " + getCommittedTaskPath(context).getParent());
    final TaskAttemptID taid = context.getTaskAttemptID();
    final TaskID tid = taid.getTaskID();
    Path pCommit = getCommittedTaskPath(context).getParent();
    // remove any committed output
    for (int i = 0; i < taid.getId(); ++i) {
        TaskAttemptID oldId = new TaskAttemptID(tid, i);
        Path pTask = new Path(pCommit, oldId.toString());
        if (!fs.delete(pTask, true) && fs.exists(pTask)) {
            throw new IOException("Failed to delete " + pTask);
        }
    }
}

19 View Complete Implementation : AbstractS3ACommitterFactory.java
Copyright Apache License 2.0
Author : apache
/**
 * Implementation point: create a task committer for a specific filesystem.
 * @param fileSystem destination FS.
 * @param outputPath final output path for work
 * @param context task context
 * @return a committer
 * @throws IOException any problem, including the FS not supporting
 * the desired committer
 */
public abstract PathOutputCommitter createTaskCommitter(S3AFileSystem fileSystem, Path outputPath, TaskAttemptContext context) throws IOException;

19 View Complete Implementation : MagicS3GuardCommitter.java
Copyright Apache License 2.0
Author : apache
/**
 * Compute the path where the output of a task attempt is stored until
 * that task is committed.
 *
 * @param context the context of the task attempt.
 * @return the path where a task attempt should be stored.
 */
public Path getTaskAttemptPath(TaskAttemptContext context) {
    return getMagicTaskAttemptPath(context, getOutputPath());
}

19 View Complete Implementation : TextOutputFormat.java
Copyright Apache License 2.0
Author : apache
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext job) throws IOException, InterruptedException {
    Configuration conf = job.getConfiguration();
    boolean isCompressed = getCompressOutput(job);
    String keyValueSeparator = conf.get(SEPARATOR, "\t");
    CompressionCodec codec = null;
    String extension = "";
    if (isCompressed) {
        Clreplaced<? extends CompressionCodec> codecClreplaced = getOutputCompressorClreplaced(job, GzipCodec.clreplaced);
        codec = ReflectionUtils.newInstance(codecClreplaced, conf);
        extension = codec.getDefaultExtension();
    }
    Path file = getDefaultWorkFile(job, extension);
    FileSystem fs = file.getFileSystem(conf);
    FSDataOutputStream fileOut = fs.create(file, false);
    if (isCompressed) {
        return new LineRecordWriter<>(new DataOutputStream(codec.createOutputStream(fileOut)), keyValueSeparator);
    } else {
        return new LineRecordWriter<>(fileOut, keyValueSeparator);
    }
}

19 View Complete Implementation : S3ACommitterFactory.java
Copyright Apache License 2.0
Author : apache
/**
 * Create a task committer.
 * @param fileSystem destination FS.
 * @param outputPath final output path for work
 * @param context job context
 * @return a committer
 * @throws IOException instantiation failure
 */
@Override
public PathOutputCommitter createTaskCommitter(S3AFileSystem fileSystem, Path outputPath, TaskAttemptContext context) throws IOException {
    AbstractS3ACommitterFactory factory = chooseCommitterFactory(fileSystem, outputPath, context.getConfiguration());
    if (factory != null) {
        PathOutputCommitter committer = factory.createTaskCommitter(fileSystem, outputPath, context);
        LOG.info("Using committer {} to output data to {}", (committer instanceof AbstractS3ACommitter ? ((AbstractS3ACommitter) committer).getName() : committer.toString()), outputPath);
        return committer;
    } else {
        LOG.warn("Using standard FileOutputCommitter to commit work." + " This is slow and potentially unsafe.");
        return createFileOutputCommitter(outputPath, context);
    }
}

19 View Complete Implementation : PathOutputCommitterFactory.java
Copyright Apache License 2.0
Author : apache
/**
 * Create the committer factory for a task attempt and destination, then
 * create the committer from it.
 * @param outputPath the task's output path, or or null if no output path
 * has been defined.
 * @param context the task attempt context
 * @return the committer to use
 * @throws IOException problems instantiating the committer
 */
public static PathOutputCommitter createCommitter(Path outputPath, TaskAttemptContext context) throws IOException {
    return getCommitterFactory(outputPath, context.getConfiguration()).createOutputCommitter(outputPath, context);
}

19 View Complete Implementation : AbstractITCommitProtocol.java
Copyright Apache License 2.0
Author : apache
/**
 * Abort a job quietly: first task, then job.
 * @param committer committer
 * @param jContext job context
 * @param tContext task context
 */
protected void abortJobQuietly(AbstractS3ACommitter committer, JobContext jContext, TaskAttemptContext tContext) {
    describe("\naborting task");
    try {
        committer.abortTask(tContext);
    } catch (IOException e) {
        log().warn("Exception aborting task:", e);
    }
    describe("\naborting job");
    try {
        committer.abortJob(jContext, JobStatus.State.KILLED);
    } catch (IOException e) {
        log().warn("Exception aborting job", e);
    }
}

19 View Complete Implementation : AbstractS3ACommitter.java
Copyright Apache License 2.0
Author : apache
/**
 * Compute the path where the output of a task attempt is stored until
 * that task is committed. This may be the normal Task attempt path
 * or it may be a subdirectory.
 * The default implementation returns the value of
 * {@link #getBaseTaskAttemptPath(TaskAttemptContext)};
 * subclreplacedes may return different values.
 * @param context the context of the task attempt.
 * @return the path where a task attempt should be stored.
 */
public Path getTaskAttemptPath(TaskAttemptContext context) {
    return getBaseTaskAttemptPath(context);
}

19 View Complete Implementation : AbstractS3ACommitterFactory.java
Copyright Apache License 2.0
Author : apache
@Override
public PathOutputCommitter createOutputCommitter(Path outputPath, TaskAttemptContext context) throws IOException {
    FileSystem fs = getDestinationFileSystem(outputPath, context);
    PathOutputCommitter outputCommitter;
    if (fs instanceof S3AFileSystem) {
        outputCommitter = createTaskCommitter((S3AFileSystem) fs, outputPath, context);
    } else {
        throw new PathCommitException(outputPath, "Filesystem not supported by this committer");
    }
    LOG.info("Using Committer {} for {}", outputCommitter, outputPath);
    return outputCommitter;
}

19 View Complete Implementation : MapFileOutputFormat.java
Copyright Apache License 2.0
Author : apache
public RecordWriter<WritableComparable<?>, Writable> getRecordWriter(TaskAttemptContext context) throws IOException {
    Configuration conf = context.getConfiguration();
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(context)) {
        // find the kind of compression to do
        compressionType = SequenceFileOutputFormat.getOutputCompressionType(context);
        // find the right codec
        Clreplaced<?> codecClreplaced = getOutputCompressorClreplaced(context, DefaultCodec.clreplaced);
        codec = (CompressionCodec) ReflectionUtils.newInstance(codecClreplaced, conf);
    }
    Path file = getDefaultWorkFile(context, "");
    FileSystem fs = file.getFileSystem(conf);
    // ignore the progress parameter, since MapFile is local
    final MapFile.Writer out = new MapFile.Writer(conf, fs, file.toString(), context.getOutputKeyClreplaced().replacedubclreplaced(WritableComparable.clreplaced), context.getOutputValueClreplaced().replacedubclreplaced(Writable.clreplaced), compressionType, codec, context);
    return new RecordWriter<WritableComparable<?>, Writable>() {

        public void write(WritableComparable<?> key, Writable value) throws IOException {
            out.append(key, value);
        }

        public void close(TaskAttemptContext context) throws IOException {
            out.close();
        }
    };
}

19 View Complete Implementation : CompositeRecordReader.java
Copyright Apache License 2.0
Author : apache
@SuppressWarnings("unchecked")
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    if (kids != null) {
        for (int i = 0; i < kids.length; ++i) {
            kids[i].initialize(((CompositeInputSplit) split).get(i), context);
            if (kids[i].key() == null) {
                continue;
            }
            // get keyclreplaced
            if (keyclreplaced == null) {
                keyclreplaced = kids[i].createKey().getClreplaced().replacedubclreplaced(WritableComparable.clreplaced);
            }
            // create priority queue
            if (null == q) {
                cmp = WritableComparator.get(keyclreplaced, conf);
                q = new PriorityQueue<ComposableRecordReader<K, ?>>(3, new Comparator<ComposableRecordReader<K, ?>>() {

                    public int compare(ComposableRecordReader<K, ?> o1, ComposableRecordReader<K, ?> o2) {
                        return cmp.compare(o1.key(), o2.key());
                    }
                });
            }
            // Explicit check for key clreplaced agreement
            if (!keyclreplaced.equals(kids[i].key().getClreplaced())) {
                throw new ClreplacedCastException("Child key clreplacedes fail to agree");
            }
            // add the kid to priority queue if it has any elements
            if (kids[i].hasNext()) {
                q.add(kids[i]);
            }
        }
    }
}

19 View Complete Implementation : ITestStagingCommitProtocol.java
Copyright Apache License 2.0
Author : apache
/**
 * The staging committers always have the local FS for their work.
 * @param committer committer instance
 * @param context task attempt context
 * @throws IOException IO failure
 */
@Override
protected void validateTaskAttemptWorkingDirectory(final AbstractS3ACommitter committer, final TaskAttemptContext context) throws IOException {
    Path wd = context.getWorkingDirectory();
    replacedertEquals("file", wd.toUri().getScheme());
}