org.apache.hadoop.mapred.JobConf.setReducerClass() - java examples

Here are the examples of the java api org.apache.hadoop.mapred.JobConf.setReducerClass() taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

155 Examples 7

15 View Complete Implementation : DFSGeneralTest.java
Copyright Apache License 2.0
Author : facebookarchive
/*
   * Spawn a map-reduce jobs based on the control files 
   * generated by the writers. 
   */
private void verifyFiles(FileSystem fs) throws IOException {
    Path inputPath = new Path(input, "filelists");
    Path outputPath = new Path(dfs_output, "verify_results");
    if (!fs.exists(inputPath)) {
        System.out.println("Couldn't find " + inputPath + " Skip verification.");
        return;
    }
    System.out.println("-------------------");
    System.out.println("VERIFY FILES");
    System.out.println("-------------------");
    JobConf conf = new JobConf(fsConfig, DFSGeneralTest.clreplaced);
    conf.set(THREAD_CLreplaced_KEY, "org.apache.hadoop.mapred.GenReaderThread");
    testtype = GenReaderThread.TEST_TYPE;
    conf.set(TEST_TYPE_KEY, testtype);
    conf.setMapperClreplaced(GenMapper.clreplaced);
    conf.setReducerClreplaced(GenReduce.clreplaced);
    conf.setJobName(getUniqueName("gentest-verify-" + testtype));
    output = getUniqueName(OUTPUT + testtype);
    updateJobConf(conf, inputPath, outputPath);
    long startTime = System.currentTimeMillis();
    JobClient.runJob(conf);
    long endTime = System.currentTimeMillis();
    printResult(fs, new Path(output, "results"), startTime, endTime);
}

15 View Complete Implementation : UtilsForTests.java
Copyright Apache License 2.0
Author : facebookarchive
// Run a job that will be killed and wait until it completes
static RunningJob runJobKill(JobConf conf, Path inDir, Path outDir) throws IOException {
    conf.setJobName("test-job-kill");
    conf.setMapperClreplaced(KillMapper.clreplaced);
    conf.setReducerClreplaced(IdenreplacedyReducer.clreplaced);
    RunningJob job = UtilsForTests.runJob(conf, inDir, outDir);
    while (job.getJobState() != JobStatus.RUNNING) {
        try {
            Thread.sleep(100);
        } catch (InterruptedException e) {
            break;
        }
    }
    job.killJob();
    while (job.cleanupProgress() == 0.0f) {
        try {
            Thread.sleep(10);
        } catch (InterruptedException ie) {
            break;
        }
    }
    return job;
}

15 View Complete Implementation : UtilsForTests.java
Copyright Apache License 2.0
Author : facebookarchive
// Run a job that will be succeeded and wait until it completes
static RunningJob runJobSucceed(JobConf conf, Path inDir, Path outDir) throws IOException {
    conf.setJobName("test-job-succeed");
    conf.setMapperClreplaced(IdenreplacedyMapper.clreplaced);
    conf.setReducerClreplaced(IdenreplacedyReducer.clreplaced);
    RunningJob job = UtilsForTests.runJob(conf, inDir, outDir);
    while (!job.isComplete()) {
        try {
            Thread.sleep(100);
        } catch (InterruptedException e) {
            break;
        }
    }
    return job;
}

15 View Complete Implementation : UtilsForTests.java
Copyright Apache License 2.0
Author : facebookarchive
// Run a job that will be failed and wait until it completes
static RunningJob runJobFail(JobConf conf, Path inDir, Path outDir) throws IOException {
    conf.setJobName("test-job-fail");
    conf.setMapperClreplaced(FailMapper.clreplaced);
    conf.setReducerClreplaced(IdenreplacedyReducer.clreplaced);
    RunningJob job = UtilsForTests.runJob(conf, inDir, outDir);
    while (!job.isComplete()) {
        try {
            Thread.sleep(100);
        } catch (InterruptedException e) {
            break;
        }
    }
    return job;
}

14 View Complete Implementation : ExternalMapReduce.java
Copyright Apache License 2.0
Author : facebookarchive
public int run(String[] argv) throws IOException {
    if (argv.length < 2) {
        System.out.println("ExternalMapReduce <input> <output>");
        return -1;
    }
    Path outDir = new Path(argv[1]);
    Path input = new Path(argv[0]);
    JobConf testConf = new JobConf(getConf(), ExternalMapReduce.clreplaced);
    // try to load a clreplaced from libjar
    try {
        testConf.getClreplacedByName("testjar.ClreplacedWordCount");
    } catch (ClreplacedNotFoundException e) {
        System.out.println("Could not find clreplaced from libjar");
        return -1;
    }
    testConf.setJobName("external job");
    FileInputFormat.setInputPaths(testConf, input);
    FileOutputFormat.setOutputPath(testConf, outDir);
    testConf.setMapperClreplaced(MapClreplaced.clreplaced);
    testConf.setReducerClreplaced(Reduce.clreplaced);
    testConf.setNumReduceTasks(1);
    JobClient.runJob(testConf);
    return 0;
}

14 View Complete Implementation : ExternalMapReduce.java
Copyright Apache License 2.0
Author : iVCE
public int run(String[] argv) throws IOException {
    if (argv.length < 2) {
        System.out.println("ExternalMapReduce <input> <output>");
        return -1;
    }
    Path outDir = new Path(argv[1]);
    Path input = new Path(argv[0]);
    JobConf testConf = new JobConf(getConf(), ExternalMapReduce.clreplaced);
    // try to load a clreplaced from libjar
    try {
        testConf.getClreplacedByName("testjar.ClreplacedWordCount");
    } catch (ClreplacedNotFoundException e) {
        System.out.println("Could not find clreplaced from libjar");
        return -1;
    }
    testConf.setJobName("external job");
    FileInputFormat.setInputPaths(testConf, input);
    FileOutputFormat.setOutputPath(testConf, outDir);
    testConf.setMapperClreplaced(MapClreplaced.clreplaced);
    testConf.setReducerClreplaced(Reduce.clreplaced);
    testConf.setNumReduceTasks(1);
    JobClient.runJob(testConf);
    return 0;
}

11 View Complete Implementation : MultiFileWordCount.java
Copyright Apache License 2.0
Author : iVCE
public int run(String[] args) throws Exception {
    if (args.length < 2) {
        printUsage();
        return 1;
    }
    JobConf job = new JobConf(getConf(), MultiFileWordCount.clreplaced);
    job.setJobName("MultiFileWordCount");
    // set the InputFormat of the job to our InputFormat
    job.setInputFormat(MyInputFormat.clreplaced);
    // the keys are words (strings)
    job.setOutputKeyClreplaced(Text.clreplaced);
    // the values are counts (ints)
    job.setOutputValueClreplaced(IntWritable.clreplaced);
    // use the defined mapper
    job.setMapperClreplaced(MapClreplaced.clreplaced);
    // use the WordCount Reducer
    job.setCombinerClreplaced(LongSumReducer.clreplaced);
    job.setReducerClreplaced(LongSumReducer.clreplaced);
    FileInputFormat.addInputPaths(job, args[0]);
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    JobClient.runJob(job);
    return 0;
}

11 View Complete Implementation : TestFileSystem.java
Copyright Apache License 2.0
Author : iVCE
public static void readTest(FileSystem fs, boolean fastCheck) throws Exception {
    fs.delete(READ_DIR, true);
    JobConf job = new JobConf(conf, TestFileSystem.clreplaced);
    job.setBoolean("fs.test.fastCheck", fastCheck);
    FileInputFormat.setInputPaths(job, CONTROL_DIR);
    job.setInputFormat(SequenceFileInputFormat.clreplaced);
    job.setMapperClreplaced(ReadMapper.clreplaced);
    job.setReducerClreplaced(LongSumReducer.clreplaced);
    FileOutputFormat.setOutputPath(job, READ_DIR);
    job.setOutputKeyClreplaced(UTF8.clreplaced);
    job.setOutputValueClreplaced(LongWritable.clreplaced);
    job.setNumReduceTasks(1);
    JobClient.runJob(job);
}

11 View Complete Implementation : MultiFileWordCount.java
Copyright Apache License 2.0
Author : madiator
public int run(String[] args) throws Exception {
    if (args.length < 2) {
        printUsage();
        return 1;
    }
    JobConf job = new JobConf(getConf(), MultiFileWordCount.clreplaced);
    job.setJobName("MultiFileWordCount");
    // set the InputFormat of the job to our InputFormat
    job.setInputFormat(MyInputFormat.clreplaced);
    // the keys are words (strings)
    job.setOutputKeyClreplaced(Text.clreplaced);
    // the values are counts (ints)
    job.setOutputValueClreplaced(IntWritable.clreplaced);
    // use the defined mapper
    job.setMapperClreplaced(MapClreplaced.clreplaced);
    // use the WordCount Reducer
    job.setCombinerClreplaced(LongSumReducer.clreplaced);
    job.setReducerClreplaced(LongSumReducer.clreplaced);
    FileInputFormat.addInputPaths(job, args[0]);
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    JobClient.runJob(job);
    return 0;
}

11 View Complete Implementation : TestFileSystem.java
Copyright Apache License 2.0
Author : facebookarchive
public static void seekTest(FileSystem fs, boolean fastCheck) throws Exception {
    fs.delete(READ_DIR, true);
    JobConf job = new JobConf(conf, TestFileSystem.clreplaced);
    job.setBoolean("fs.test.fastCheck", fastCheck);
    FileInputFormat.setInputPaths(job, CONTROL_DIR);
    job.setInputFormat(SequenceFileInputFormat.clreplaced);
    job.setMapperClreplaced(SeekMapper.clreplaced);
    job.setReducerClreplaced(LongSumReducer.clreplaced);
    FileOutputFormat.setOutputPath(job, READ_DIR);
    job.setOutputKeyClreplaced(UTF8.clreplaced);
    job.setOutputValueClreplaced(LongWritable.clreplaced);
    job.setNumReduceTasks(1);
    JobClient.runJob(job);
}

11 View Complete Implementation : TestFileSystem.java
Copyright Apache License 2.0
Author : madiator
public static void readTest(FileSystem fs, boolean fastCheck) throws Exception {
    fs.delete(READ_DIR, true);
    JobConf job = new JobConf(conf, TestFileSystem.clreplaced);
    job.setBoolean("fs.test.fastCheck", fastCheck);
    FileInputFormat.setInputPaths(job, CONTROL_DIR);
    job.setInputFormat(SequenceFileInputFormat.clreplaced);
    job.setMapperClreplaced(ReadMapper.clreplaced);
    job.setReducerClreplaced(LongSumReducer.clreplaced);
    FileOutputFormat.setOutputPath(job, READ_DIR);
    job.setOutputKeyClreplaced(UTF8.clreplaced);
    job.setOutputValueClreplaced(LongWritable.clreplaced);
    job.setNumReduceTasks(1);
    JobClient.runJob(job);
}

11 View Complete Implementation : TestFileSystem.java
Copyright Apache License 2.0
Author : iVCE
public static void seekTest(FileSystem fs, boolean fastCheck) throws Exception {
    fs.delete(READ_DIR, true);
    JobConf job = new JobConf(conf, TestFileSystem.clreplaced);
    job.setBoolean("fs.test.fastCheck", fastCheck);
    FileInputFormat.setInputPaths(job, CONTROL_DIR);
    job.setInputFormat(SequenceFileInputFormat.clreplaced);
    job.setMapperClreplaced(SeekMapper.clreplaced);
    job.setReducerClreplaced(LongSumReducer.clreplaced);
    FileOutputFormat.setOutputPath(job, READ_DIR);
    job.setOutputKeyClreplaced(UTF8.clreplaced);
    job.setOutputValueClreplaced(LongWritable.clreplaced);
    job.setNumReduceTasks(1);
    JobClient.runJob(job);
}

11 View Complete Implementation : JobControlTestUtils.java
Copyright Apache License 2.0
Author : facebookarchive
/**
 * Creates a simple copy job.
 *
 * @param indirs List of input directories.
 * @param outdir Output directory.
 * @return JobConf initialised for a simple copy job.
 * @throws Exception If an error occurs creating job configuration.
 */
static JobConf createCopyJob(List<Path> indirs, Path outdir) throws Exception {
    Configuration defaults = new Configuration();
    JobConf theJob = new JobConf(defaults, TestJobControl.clreplaced);
    theJob.setJobName("DataMoveJob");
    FileInputFormat.setInputPaths(theJob, indirs.toArray(new Path[0]));
    theJob.setMapperClreplaced(DataCopy.clreplaced);
    FileOutputFormat.setOutputPath(theJob, outdir);
    theJob.setOutputKeyClreplaced(Text.clreplaced);
    theJob.setOutputValueClreplaced(Text.clreplaced);
    theJob.setReducerClreplaced(DataCopy.clreplaced);
    theJob.setNumMapTasks(12);
    theJob.setNumReduceTasks(4);
    return theJob;
}

11 View Complete Implementation : JobControlTestUtils.java
Copyright Apache License 2.0
Author : iVCE
/**
 * Creates a simple copy job.
 *
 * @param indirs List of input directories.
 * @param outdir Output directory.
 * @return JobConf initialised for a simple copy job.
 * @throws Exception If an error occurs creating job configuration.
 */
static JobConf createCopyJob(List<Path> indirs, Path outdir) throws Exception {
    Configuration defaults = new Configuration();
    JobConf theJob = new JobConf(defaults, TestJobControl.clreplaced);
    theJob.setJobName("DataMoveJob");
    FileInputFormat.setInputPaths(theJob, indirs.toArray(new Path[0]));
    theJob.setMapperClreplaced(DataCopy.clreplaced);
    FileOutputFormat.setOutputPath(theJob, outdir);
    theJob.setOutputKeyClreplaced(Text.clreplaced);
    theJob.setOutputValueClreplaced(Text.clreplaced);
    theJob.setReducerClreplaced(DataCopy.clreplaced);
    theJob.setNumMapTasks(12);
    theJob.setNumReduceTasks(4);
    return theJob;
}

11 View Complete Implementation : MultiFileWordCount.java
Copyright Apache License 2.0
Author : facebookarchive
public int run(String[] args) throws Exception {
    if (args.length < 2) {
        printUsage();
        return 1;
    }
    JobConf job = new JobConf(getConf(), MultiFileWordCount.clreplaced);
    job.setJobName("MultiFileWordCount");
    // set the InputFormat of the job to our InputFormat
    job.setInputFormat(MyInputFormat.clreplaced);
    // the keys are words (strings)
    job.setOutputKeyClreplaced(Text.clreplaced);
    // the values are counts (ints)
    job.setOutputValueClreplaced(IntWritable.clreplaced);
    // use the defined mapper
    job.setMapperClreplaced(MapClreplaced.clreplaced);
    // use the WordCount Reducer
    job.setCombinerClreplaced(LongSumReducer.clreplaced);
    job.setReducerClreplaced(LongSumReducer.clreplaced);
    FileInputFormat.addInputPaths(job, args[0]);
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    JobClient.runJob(job);
    return 0;
}

11 View Complete Implementation : TestFileSystem.java
Copyright Apache License 2.0
Author : facebookarchive
public static void writeTest(FileSystem fs, boolean fastCheck) throws Exception {
    fs.delete(DATA_DIR, true);
    fs.delete(WRITE_DIR, true);
    JobConf job = new JobConf(conf, TestFileSystem.clreplaced);
    job.setBoolean("fs.test.fastCheck", fastCheck);
    FileInputFormat.setInputPaths(job, CONTROL_DIR);
    job.setInputFormat(SequenceFileInputFormat.clreplaced);
    job.setMapperClreplaced(WriteMapper.clreplaced);
    job.setReducerClreplaced(LongSumReducer.clreplaced);
    FileOutputFormat.setOutputPath(job, WRITE_DIR);
    job.setOutputKeyClreplaced(UTF8.clreplaced);
    job.setOutputValueClreplaced(LongWritable.clreplaced);
    job.setNumReduceTasks(1);
    JobClient.runJob(job);
}

11 View Complete Implementation : TestFileSystem.java
Copyright Apache License 2.0
Author : facebookarchive
public static void readTest(FileSystem fs, boolean fastCheck) throws Exception {
    fs.delete(READ_DIR, true);
    JobConf job = new JobConf(conf, TestFileSystem.clreplaced);
    job.setBoolean("fs.test.fastCheck", fastCheck);
    FileInputFormat.setInputPaths(job, CONTROL_DIR);
    job.setInputFormat(SequenceFileInputFormat.clreplaced);
    job.setMapperClreplaced(ReadMapper.clreplaced);
    job.setReducerClreplaced(LongSumReducer.clreplaced);
    FileOutputFormat.setOutputPath(job, READ_DIR);
    job.setOutputKeyClreplaced(UTF8.clreplaced);
    job.setOutputValueClreplaced(LongWritable.clreplaced);
    job.setNumReduceTasks(1);
    JobClient.runJob(job);
}

11 View Complete Implementation : TestFileSystem.java
Copyright Apache License 2.0
Author : iVCE
public static void writeTest(FileSystem fs, boolean fastCheck) throws Exception {
    fs.delete(DATA_DIR, true);
    fs.delete(WRITE_DIR, true);
    JobConf job = new JobConf(conf, TestFileSystem.clreplaced);
    job.setBoolean("fs.test.fastCheck", fastCheck);
    FileInputFormat.setInputPaths(job, CONTROL_DIR);
    job.setInputFormat(SequenceFileInputFormat.clreplaced);
    job.setMapperClreplaced(WriteMapper.clreplaced);
    job.setReducerClreplaced(LongSumReducer.clreplaced);
    FileOutputFormat.setOutputPath(job, WRITE_DIR);
    job.setOutputKeyClreplaced(UTF8.clreplaced);
    job.setOutputValueClreplaced(LongWritable.clreplaced);
    job.setNumReduceTasks(1);
    JobClient.runJob(job);
}

11 View Complete Implementation : TestFileSystem.java
Copyright Apache License 2.0
Author : madiator
public static void seekTest(FileSystem fs, boolean fastCheck) throws Exception {
    fs.delete(READ_DIR, true);
    JobConf job = new JobConf(conf, TestFileSystem.clreplaced);
    job.setBoolean("fs.test.fastCheck", fastCheck);
    FileInputFormat.setInputPaths(job, CONTROL_DIR);
    job.setInputFormat(SequenceFileInputFormat.clreplaced);
    job.setMapperClreplaced(SeekMapper.clreplaced);
    job.setReducerClreplaced(LongSumReducer.clreplaced);
    FileOutputFormat.setOutputPath(job, READ_DIR);
    job.setOutputKeyClreplaced(UTF8.clreplaced);
    job.setOutputValueClreplaced(LongWritable.clreplaced);
    job.setNumReduceTasks(1);
    JobClient.runJob(job);
}

11 View Complete Implementation : JobControlTestUtils.java
Copyright Apache License 2.0
Author : madiator
/**
 * Creates a simple copy job.
 *
 * @param indirs List of input directories.
 * @param outdir Output directory.
 * @return JobConf initialised for a simple copy job.
 * @throws Exception If an error occurs creating job configuration.
 */
static JobConf createCopyJob(List<Path> indirs, Path outdir) throws Exception {
    Configuration defaults = new Configuration();
    JobConf theJob = new JobConf(defaults, TestJobControl.clreplaced);
    theJob.setJobName("DataMoveJob");
    FileInputFormat.setInputPaths(theJob, indirs.toArray(new Path[0]));
    theJob.setMapperClreplaced(DataCopy.clreplaced);
    FileOutputFormat.setOutputPath(theJob, outdir);
    theJob.setOutputKeyClreplaced(Text.clreplaced);
    theJob.setOutputValueClreplaced(Text.clreplaced);
    theJob.setReducerClreplaced(DataCopy.clreplaced);
    theJob.setNumMapTasks(12);
    theJob.setNumReduceTasks(4);
    return theJob;
}

11 View Complete Implementation : TestFileSystem.java
Copyright Apache License 2.0
Author : madiator
public static void writeTest(FileSystem fs, boolean fastCheck) throws Exception {
    fs.delete(DATA_DIR, true);
    fs.delete(WRITE_DIR, true);
    JobConf job = new JobConf(conf, TestFileSystem.clreplaced);
    job.setBoolean("fs.test.fastCheck", fastCheck);
    FileInputFormat.setInputPaths(job, CONTROL_DIR);
    job.setInputFormat(SequenceFileInputFormat.clreplaced);
    job.setMapperClreplaced(WriteMapper.clreplaced);
    job.setReducerClreplaced(LongSumReducer.clreplaced);
    FileOutputFormat.setOutputPath(job, WRITE_DIR);
    job.setOutputKeyClreplaced(UTF8.clreplaced);
    job.setOutputValueClreplaced(LongWritable.clreplaced);
    job.setNumReduceTasks(1);
    JobClient.runJob(job);
}

10 View Complete Implementation : TeraValidate.java
Copyright Apache License 2.0
Author : facebookarchive
public int run(String[] args) throws Exception {
    JobConf job = (JobConf) getConf();
    TeraInputFormat.setInputPaths(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    job.setJobName("TeraValidate");
    job.setJarByClreplaced(TeraValidate.clreplaced);
    job.setMapperClreplaced(ValidateMapper.clreplaced);
    job.setReducerClreplaced(ValidateReducer.clreplaced);
    job.setOutputKeyClreplaced(Text.clreplaced);
    job.setOutputValueClreplaced(Text.clreplaced);
    // force a single reducer
    job.setNumReduceTasks(1);
    // force a single split
    job.setLong("mapred.min.split.size", Long.MAX_VALUE);
    job.setInputFormat(TeraInputFormat.clreplaced);
    JobClient.runJob(job);
    return 0;
}

10 View Complete Implementation : TestDatamerge.java
Copyright Apache License 2.0
Author : facebookarchive
public void testEmptyJoin() throws Exception {
    JobConf job = new JobConf();
    Path base = cluster.getFileSystem().makeQualified(new Path("/empty"));
    Path[] src = { new Path(base, "i0"), new Path("i1"), new Path("i2") };
    job.set("mapred.join.expr", CompositeInputFormat.compose("outer", Fake_IF.clreplaced, src));
    job.setInputFormat(CompositeInputFormat.clreplaced);
    FileOutputFormat.setOutputPath(job, new Path(base, "out"));
    job.setMapperClreplaced(IdenreplacedyMapper.clreplaced);
    job.setReducerClreplaced(IdenreplacedyReducer.clreplaced);
    job.setOutputKeyClreplaced(IncomparableKey.clreplaced);
    job.setOutputValueClreplaced(NullWritable.clreplaced);
    JobClient.runJob(job);
    base.getFileSystem(job).delete(base, true);
}

10 View Complete Implementation : TeraValidate.java
Copyright Apache License 2.0
Author : iVCE
public int run(String[] args) throws Exception {
    JobConf job = (JobConf) getConf();
    TeraInputFormat.setInputPaths(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    job.setJobName("TeraValidate");
    job.setJarByClreplaced(TeraValidate.clreplaced);
    job.setMapperClreplaced(ValidateMapper.clreplaced);
    job.setReducerClreplaced(ValidateReducer.clreplaced);
    job.setOutputKeyClreplaced(Text.clreplaced);
    job.setOutputValueClreplaced(Text.clreplaced);
    // force a single reducer
    job.setNumReduceTasks(1);
    // force a single split
    job.setLong("mapred.min.split.size", Long.MAX_VALUE);
    job.setInputFormat(TeraInputFormat.clreplaced);
    JobClient.runJob(job);
    return 0;
}

10 View Complete Implementation : TeraValidate.java
Copyright Apache License 2.0
Author : madiator
public int run(String[] args) throws Exception {
    JobConf job = (JobConf) getConf();
    TeraInputFormat.setInputPaths(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    job.setJobName("TeraValidate");
    job.setJarByClreplaced(TeraValidate.clreplaced);
    job.setMapperClreplaced(ValidateMapper.clreplaced);
    job.setReducerClreplaced(ValidateReducer.clreplaced);
    job.setOutputKeyClreplaced(Text.clreplaced);
    job.setOutputValueClreplaced(Text.clreplaced);
    // force a single reducer
    job.setNumReduceTasks(1);
    // force a single split
    job.setLong("mapred.min.split.size", Long.MAX_VALUE);
    job.setInputFormat(TeraInputFormat.clreplaced);
    JobClient.runJob(job);
    return 0;
}

10 View Complete Implementation : TestDatamerge.java
Copyright Apache License 2.0
Author : iVCE
public void testEmptyJoin() throws Exception {
    JobConf job = new JobConf();
    Path base = cluster.getFileSystem().makeQualified(new Path("/empty"));
    Path[] src = { new Path(base, "i0"), new Path("i1"), new Path("i2") };
    job.set("mapred.join.expr", CompositeInputFormat.compose("outer", Fake_IF.clreplaced, src));
    job.setInputFormat(CompositeInputFormat.clreplaced);
    FileOutputFormat.setOutputPath(job, new Path(base, "out"));
    job.setMapperClreplaced(IdenreplacedyMapper.clreplaced);
    job.setReducerClreplaced(IdenreplacedyReducer.clreplaced);
    job.setOutputKeyClreplaced(IncomparableKey.clreplaced);
    job.setOutputValueClreplaced(NullWritable.clreplaced);
    JobClient.runJob(job);
    base.getFileSystem(job).delete(base, true);
}

10 View Complete Implementation : TestDatamerge.java
Copyright Apache License 2.0
Author : madiator
public void testEmptyJoin() throws Exception {
    JobConf job = new JobConf();
    Path base = cluster.getFileSystem().makeQualified(new Path("/empty"));
    Path[] src = { new Path(base, "i0"), new Path("i1"), new Path("i2") };
    job.set("mapred.join.expr", CompositeInputFormat.compose("outer", Fake_IF.clreplaced, src));
    job.setInputFormat(CompositeInputFormat.clreplaced);
    FileOutputFormat.setOutputPath(job, new Path(base, "out"));
    job.setMapperClreplaced(IdenreplacedyMapper.clreplaced);
    job.setReducerClreplaced(IdenreplacedyReducer.clreplaced);
    job.setOutputKeyClreplaced(IncomparableKey.clreplaced);
    job.setOutputValueClreplaced(NullWritable.clreplaced);
    JobClient.runJob(job);
    base.getFileSystem(job).delete(base, true);
}

9 View Complete Implementation : TestDatamerge.java
Copyright Apache License 2.0
Author : facebookarchive
private static void joinAs(String jointype, Clreplaced<? extends SimpleCheckerBase> c) throws Exception {
    final int srcs = 4;
    Configuration conf = new Configuration();
    JobConf job = new JobConf(conf, c);
    Path base = cluster.getFileSystem().makeQualified(new Path("/" + jointype));
    Path[] src = writeSimpleSrc(base, conf, srcs);
    job.set("mapred.join.expr", CompositeInputFormat.compose(jointype, SequenceFileInputFormat.clreplaced, src));
    job.setInt("testdatamerge.sources", srcs);
    job.setInputFormat(CompositeInputFormat.clreplaced);
    FileOutputFormat.setOutputPath(job, new Path(base, "out"));
    job.setMapperClreplaced(c);
    job.setReducerClreplaced(c);
    job.setOutputKeyClreplaced(IntWritable.clreplaced);
    job.setOutputValueClreplaced(IntWritable.clreplaced);
    JobClient.runJob(job);
    base.getFileSystem(job).delete(base, true);
}

9 View Complete Implementation : TestDatamerge.java
Copyright Apache License 2.0
Author : madiator
private static void joinAs(String jointype, Clreplaced<? extends SimpleCheckerBase> c) throws Exception {
    final int srcs = 4;
    Configuration conf = new Configuration();
    JobConf job = new JobConf(conf, c);
    Path base = cluster.getFileSystem().makeQualified(new Path("/" + jointype));
    Path[] src = writeSimpleSrc(base, conf, srcs);
    job.set("mapred.join.expr", CompositeInputFormat.compose(jointype, SequenceFileInputFormat.clreplaced, src));
    job.setInt("testdatamerge.sources", srcs);
    job.setInputFormat(CompositeInputFormat.clreplaced);
    FileOutputFormat.setOutputPath(job, new Path(base, "out"));
    job.setMapperClreplaced(c);
    job.setReducerClreplaced(c);
    job.setOutputKeyClreplaced(IntWritable.clreplaced);
    job.setOutputValueClreplaced(IntWritable.clreplaced);
    JobClient.runJob(job);
    base.getFileSystem(job).delete(base, true);
}

9 View Complete Implementation : TestDatamerge.java
Copyright Apache License 2.0
Author : iVCE
private static void joinAs(String jointype, Clreplaced<? extends SimpleCheckerBase> c) throws Exception {
    final int srcs = 4;
    Configuration conf = new Configuration();
    JobConf job = new JobConf(conf, c);
    Path base = cluster.getFileSystem().makeQualified(new Path("/" + jointype));
    Path[] src = writeSimpleSrc(base, conf, srcs);
    job.set("mapred.join.expr", CompositeInputFormat.compose(jointype, SequenceFileInputFormat.clreplaced, src));
    job.setInt("testdatamerge.sources", srcs);
    job.setInputFormat(CompositeInputFormat.clreplaced);
    FileOutputFormat.setOutputPath(job, new Path(base, "out"));
    job.setMapperClreplaced(c);
    job.setReducerClreplaced(c);
    job.setOutputKeyClreplaced(IntWritable.clreplaced);
    job.setOutputValueClreplaced(IntWritable.clreplaced);
    JobClient.runJob(job);
    base.getFileSystem(job).delete(base, true);
}

8 View Complete Implementation : ComputeJob.java
Copyright Apache License 2.0
Author : facebookarchive
public JobConf setupJobConf(int numMapper, long round, int base, int step, String output) {
    JobConf job = new JobConf(getConf(), ComputeJob.clreplaced);
    job.setJobSetupCleanupNeeded(true);
    job.setNumMapTasks(numMapper);
    job.setNumReduceTasks(1);
    job.setMapperClreplaced(ComputeJob.clreplaced);
    job.setMapOutputKeyClreplaced(IntWritable.clreplaced);
    job.setMapOutputValueClreplaced(Text.clreplaced);
    job.setReducerClreplaced(ComputeJob.clreplaced);
    job.setOutputFormat(TextOutputFormat.clreplaced);
    job.setInputFormat(ComputeInputFormat.clreplaced);
    job.setJobName("Compute job");
    FileInputFormat.addInputPath(job, new Path("ignored"));
    Path outputPath = new Path(output);
    try {
        FileSystem dfs = outputPath.getFileSystem(job);
        if (dfs.exists(outputPath))
            dfs.delete(outputPath, true);
    } catch (IOException e) {
        e.printStackTrace();
    }
    FileOutputFormat.setOutputPath(job, outputPath);
    job.setLong("compute.job.round", round);
    job.setLong("compute.job.base", base);
    job.setLong("compute.job.step", step);
    job.setSpeculativeExecution(false);
    return job;
}

8 View Complete Implementation : UtilsForTests.java
Copyright Apache License 2.0
Author : koichi626
/**
 * Configure a waiting job
 */
static void configureWaitingJobConf(JobConf jobConf, Path inDir, Path outputPath, int numMaps, int numRed, String jobName, String mapSignalFilename, String redSignalFilename) throws IOException {
    jobConf.setJobName(jobName);
    jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.clreplaced);
    jobConf.setOutputFormat(SequenceFileOutputFormat.clreplaced);
    FileInputFormat.setInputPaths(jobConf, inDir);
    FileOutputFormat.setOutputPath(jobConf, outputPath);
    jobConf.setMapperClreplaced(UtilsForTests.HalfWaitingMapper.clreplaced);
    jobConf.setReducerClreplaced(IdenreplacedyReducer.clreplaced);
    jobConf.setOutputKeyClreplaced(BytesWritable.clreplaced);
    jobConf.setOutputValueClreplaced(BytesWritable.clreplaced);
    jobConf.setInputFormat(RandomInputFormat.clreplaced);
    jobConf.setNumMapTasks(numMaps);
    jobConf.setNumReduceTasks(numRed);
    jobConf.setJar("build/test/testjar/testjob.jar");
    jobConf.set(getTaskSignalParameter(true), mapSignalFilename);
    jobConf.set(getTaskSignalParameter(false), redSignalFilename);
}

8 View Complete Implementation : UtilsForTests.java
Copyright Apache License 2.0
Author : iVCE
/**
 * Configure a waiting job
 */
static void configureWaitingJobConf(JobConf jobConf, Path inDir, Path outputPath, int numMaps, int numRed, String jobName, String mapSignalFilename, String redSignalFilename) throws IOException {
    jobConf.setJobName(jobName);
    jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.clreplaced);
    jobConf.setOutputFormat(SequenceFileOutputFormat.clreplaced);
    FileInputFormat.setInputPaths(jobConf, inDir);
    FileOutputFormat.setOutputPath(jobConf, outputPath);
    jobConf.setMapperClreplaced(UtilsForTests.HalfWaitingMapper.clreplaced);
    jobConf.setReducerClreplaced(IdenreplacedyReducer.clreplaced);
    jobConf.setOutputKeyClreplaced(BytesWritable.clreplaced);
    jobConf.setOutputValueClreplaced(BytesWritable.clreplaced);
    jobConf.setInputFormat(RandomInputFormat.clreplaced);
    jobConf.setNumMapTasks(numMaps);
    jobConf.setNumReduceTasks(numRed);
    jobConf.setJar("build/test/testjar/testjob.jar");
    jobConf.set(getTaskSignalParameter(true), mapSignalFilename);
    jobConf.set(getTaskSignalParameter(false), redSignalFilename);
}

8 View Complete Implementation : TransposeJob.java
Copyright Apache License 2.0
Author : sisirkoppaka
public static JobConf buildTransposeJobConf(Path matrixInputPath, Path matrixOutputPath, int numInputRows) throws IOException {
    JobConf conf = new JobConf(TransposeJob.clreplaced);
    conf.setJobName("TransposeJob: " + matrixInputPath + " transpose -> " + matrixOutputPath);
    FileSystem fs = FileSystem.get(conf);
    matrixInputPath = fs.makeQualified(matrixInputPath);
    matrixOutputPath = fs.makeQualified(matrixOutputPath);
    conf.setInt(NUM_ROWS_KEY, numInputRows);
    FileInputFormat.addInputPath(conf, matrixInputPath);
    conf.setInputFormat(SequenceFileInputFormat.clreplaced);
    FileOutputFormat.setOutputPath(conf, matrixOutputPath);
    conf.setMapperClreplaced(TransposeMapper.clreplaced);
    conf.setReducerClreplaced(TransposeReducer.clreplaced);
    conf.setMapOutputKeyClreplaced(IntWritable.clreplaced);
    conf.setMapOutputValueClreplaced(DistributedRowMatrix.MatrixEntryWritable.clreplaced);
    conf.setOutputFormat(SequenceFileOutputFormat.clreplaced);
    conf.setOutputKeyClreplaced(IntWritable.clreplaced);
    conf.setOutputValueClreplaced(VectorWritable.clreplaced);
    return conf;
}

8 View Complete Implementation : InputDriver.java
Copyright Apache License 2.0
Author : sisirkoppaka
public static void runJob(Path input, Path output, String vectorClreplacedName) throws IOException {
    JobClient client = new JobClient();
    JobConf conf = new JobConf(InputDriver.clreplaced);
    conf.setOutputKeyClreplaced(Text.clreplaced);
    conf.setOutputValueClreplaced(VectorWritable.clreplaced);
    conf.setOutputFormat(SequenceFileOutputFormat.clreplaced);
    conf.set("vector.implementation.clreplaced.name", vectorClreplacedName);
    FileInputFormat.setInputPaths(conf, input);
    FileOutputFormat.setOutputPath(conf, output);
    conf.setMapperClreplaced(InputMapper.clreplaced);
    conf.setReducerClreplaced(Reducer.clreplaced);
    conf.setNumReduceTasks(0);
    client.setConf(conf);
    JobClient.runJob(conf);
}

8 View Complete Implementation : InputDriver.java
Copyright Apache License 2.0
Author : ogrisel
public static void runJob(Path input, Path output, String vectorClreplacedName) throws IOException {
    JobClient client = new JobClient();
    JobConf conf = new JobConf(InputDriver.clreplaced);
    conf.setOutputKeyClreplaced(Text.clreplaced);
    conf.setOutputValueClreplaced(VectorWritable.clreplaced);
    conf.setOutputFormat(SequenceFileOutputFormat.clreplaced);
    conf.set("vector.implementation.clreplaced.name", vectorClreplacedName);
    FileInputFormat.setInputPaths(conf, input);
    FileOutputFormat.setOutputPath(conf, output);
    conf.setMapperClreplaced(InputMapper.clreplaced);
    conf.setReducerClreplaced(Reducer.clreplaced);
    conf.setNumReduceTasks(0);
    client.setConf(conf);
    JobClient.runJob(conf);
}

8 View Complete Implementation : MRSharedCaching.java
Copyright Apache License 2.0
Author : madiator
// Boilerplate code
public static FileSystem setupJob(String indir, String outdir, String cacheDir, JobConf conf, String input, boolean withSymlink) throws IOException {
    final Path inDir = new Path(indir);
    final Path outDir = new Path(outdir);
    FileSystem fs = FileSystem.get(conf);
    fs.delete(outDir, true);
    if (!fs.mkdirs(inDir)) {
        throw new IOException("Mkdirs failed to create " + inDir.toString());
    }
    {
        DataOutputStream file = fs.create(new Path(inDir, "part-0"));
        file.writeBytes(input);
        file.close();
    }
    conf.setJobName("sharedcachetest");
    // the keys are words (strings)
    conf.setOutputKeyClreplaced(Text.clreplaced);
    // the values are counts (ints)
    conf.setOutputValueClreplaced(IntWritable.clreplaced);
    conf.setCombinerClreplaced(MRSharedCaching.ReduceClreplaced.clreplaced);
    conf.setReducerClreplaced(MRSharedCaching.ReduceClreplaced.clreplaced);
    FileInputFormat.setInputPaths(conf, inDir);
    FileOutputFormat.setOutputPath(conf, outDir);
    conf.setNumMapTasks(1);
    conf.setNumReduceTasks(1);
    conf.setSpeculativeExecution(false);
    if (!withSymlink) {
        conf.setMapperClreplaced(MRSharedCaching.MapClreplaced.clreplaced);
    } else {
        conf.setMapperClreplaced(MRSharedCaching.MapClreplaced2.clreplaced);
    }
    // Turn on sharing
    conf.set("mapred.cache.shared.enabled", "true");
    return fs;
}

8 View Complete Implementation : TransposeJob.java
Copyright Apache License 2.0
Author : ogrisel
public static JobConf buildTransposeJobConf(Path matrixInputPath, Path matrixOutputPath, int numInputRows) throws IOException {
    JobConf conf = new JobConf(TransposeJob.clreplaced);
    conf.setJobName("TransposeJob: " + matrixInputPath + " transpose -> " + matrixOutputPath);
    FileSystem fs = FileSystem.get(conf);
    matrixInputPath = fs.makeQualified(matrixInputPath);
    matrixOutputPath = fs.makeQualified(matrixOutputPath);
    conf.setInt(NUM_ROWS_KEY, numInputRows);
    FileInputFormat.addInputPath(conf, matrixInputPath);
    conf.setInputFormat(SequenceFileInputFormat.clreplaced);
    FileOutputFormat.setOutputPath(conf, matrixOutputPath);
    conf.setMapperClreplaced(TransposeMapper.clreplaced);
    conf.setReducerClreplaced(TransposeReducer.clreplaced);
    conf.setMapOutputKeyClreplaced(IntWritable.clreplaced);
    conf.setMapOutputValueClreplaced(DistributedRowMatrix.MatrixEntryWritable.clreplaced);
    conf.setOutputFormat(SequenceFileOutputFormat.clreplaced);
    conf.setOutputKeyClreplaced(IntWritable.clreplaced);
    conf.setOutputValueClreplaced(VectorWritable.clreplaced);
    return conf;
}

8 View Complete Implementation : UtilsForTests.java
Copyright Apache License 2.0
Author : facebookarchive
/**
 * Configure a waiting job
 */
static void configureWaitingJobConf(JobConf jobConf, Path inDir, Path outputPath, int numMaps, int numRed, String jobName, String mapSignalFilename, String redSignalFilename) throws IOException {
    jobConf.setJobName(jobName);
    jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.clreplaced);
    jobConf.setOutputFormat(SequenceFileOutputFormat.clreplaced);
    FileInputFormat.setInputPaths(jobConf, inDir);
    FileOutputFormat.setOutputPath(jobConf, outputPath);
    jobConf.setMapperClreplaced(UtilsForTests.HalfWaitingMapper.clreplaced);
    jobConf.setReducerClreplaced(IdenreplacedyReducer.clreplaced);
    jobConf.setOutputKeyClreplaced(BytesWritable.clreplaced);
    jobConf.setOutputValueClreplaced(BytesWritable.clreplaced);
    jobConf.setInputFormat(RandomInputFormat.clreplaced);
    jobConf.setNumMapTasks(numMaps);
    jobConf.setNumReduceTasks(numRed);
    jobConf.setJar("build/test/testjar/testjob.jar");
    jobConf.set(getTaskSignalParameter(true), mapSignalFilename);
    jobConf.set(getTaskSignalParameter(false), redSignalFilename);
}

8 View Complete Implementation : MatrixMultiplicationJob.java
Copyright Apache License 2.0
Author : ogrisel
public static JobConf createMatrixMultiplyJobConf(Path aPath, Path bPath, Path outPath, int outCardinality) {
    JobConf conf = new JobConf(MatrixMultiplicationJob.clreplaced);
    conf.setInputFormat(CompositeInputFormat.clreplaced);
    conf.set("mapred.join.expr", CompositeInputFormat.compose("inner", SequenceFileInputFormat.clreplaced, aPath, bPath));
    conf.setInt(OUT_CARD, outCardinality);
    conf.setOutputFormat(SequenceFileOutputFormat.clreplaced);
    FileOutputFormat.setOutputPath(conf, outPath);
    conf.setMapperClreplaced(MatrixMultiplyMapper.clreplaced);
    conf.setCombinerClreplaced(MatrixMultiplicationReducer.clreplaced);
    conf.setReducerClreplaced(MatrixMultiplicationReducer.clreplaced);
    conf.setMapOutputKeyClreplaced(IntWritable.clreplaced);
    conf.setMapOutputValueClreplaced(VectorWritable.clreplaced);
    conf.setOutputKeyClreplaced(IntWritable.clreplaced);
    conf.setOutputValueClreplaced(VectorWritable.clreplaced);
    return conf;
}

8 View Complete Implementation : UtilsForTests.java
Copyright Apache License 2.0
Author : madiator
/**
 * Configure a waiting job
 */
static void configureWaitingJobConf(JobConf jobConf, Path inDir, Path outputPath, int numMaps, int numRed, String jobName, String mapSignalFilename, String redSignalFilename) throws IOException {
    jobConf.setJobName(jobName);
    jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.clreplaced);
    jobConf.setOutputFormat(SequenceFileOutputFormat.clreplaced);
    FileInputFormat.setInputPaths(jobConf, inDir);
    FileOutputFormat.setOutputPath(jobConf, outputPath);
    jobConf.setMapperClreplaced(UtilsForTests.HalfWaitingMapper.clreplaced);
    jobConf.setReducerClreplaced(IdenreplacedyReducer.clreplaced);
    jobConf.setOutputKeyClreplaced(BytesWritable.clreplaced);
    jobConf.setOutputValueClreplaced(BytesWritable.clreplaced);
    jobConf.setInputFormat(RandomInputFormat.clreplaced);
    jobConf.setNumMapTasks(numMaps);
    jobConf.setNumReduceTasks(numRed);
    jobConf.setJar("build/test/testjar/testjob.jar");
    jobConf.set(getTaskSignalParameter(true), mapSignalFilename);
    jobConf.set(getTaskSignalParameter(false), redSignalFilename);
}

8 View Complete Implementation : MatrixMultiplicationJob.java
Copyright Apache License 2.0
Author : sisirkoppaka
public static JobConf createMatrixMultiplyJobConf(Path aPath, Path bPath, Path outPath, int outCardinality) {
    JobConf conf = new JobConf(MatrixMultiplicationJob.clreplaced);
    conf.setInputFormat(CompositeInputFormat.clreplaced);
    conf.set("mapred.join.expr", CompositeInputFormat.compose("inner", SequenceFileInputFormat.clreplaced, aPath, bPath));
    conf.setInt(OUT_CARD, outCardinality);
    conf.setOutputFormat(SequenceFileOutputFormat.clreplaced);
    FileOutputFormat.setOutputPath(conf, outPath);
    conf.setMapperClreplaced(MatrixMultiplyMapper.clreplaced);
    conf.setCombinerClreplaced(MatrixMultiplicationReducer.clreplaced);
    conf.setReducerClreplaced(MatrixMultiplicationReducer.clreplaced);
    conf.setMapOutputKeyClreplaced(IntWritable.clreplaced);
    conf.setMapOutputValueClreplaced(VectorWritable.clreplaced);
    conf.setOutputKeyClreplaced(IntWritable.clreplaced);
    conf.setOutputValueClreplaced(VectorWritable.clreplaced);
    return conf;
}

8 View Complete Implementation : MRSharedCaching.java
Copyright Apache License 2.0
Author : iVCE
// Boilerplate code
public static FileSystem setupJob(String indir, String outdir, String cacheDir, JobConf conf, String input, boolean withSymlink) throws IOException {
    final Path inDir = new Path(indir);
    final Path outDir = new Path(outdir);
    FileSystem fs = FileSystem.get(conf);
    fs.delete(outDir, true);
    if (!fs.mkdirs(inDir)) {
        throw new IOException("Mkdirs failed to create " + inDir.toString());
    }
    {
        DataOutputStream file = fs.create(new Path(inDir, "part-0"));
        file.writeBytes(input);
        file.close();
    }
    conf.setJobName("sharedcachetest");
    // the keys are words (strings)
    conf.setOutputKeyClreplaced(Text.clreplaced);
    // the values are counts (ints)
    conf.setOutputValueClreplaced(IntWritable.clreplaced);
    conf.setCombinerClreplaced(MRSharedCaching.ReduceClreplaced.clreplaced);
    conf.setReducerClreplaced(MRSharedCaching.ReduceClreplaced.clreplaced);
    FileInputFormat.setInputPaths(conf, inDir);
    FileOutputFormat.setOutputPath(conf, outDir);
    conf.setNumMapTasks(1);
    conf.setNumReduceTasks(1);
    conf.setSpeculativeExecution(false);
    if (!withSymlink) {
        conf.setMapperClreplaced(MRSharedCaching.MapClreplaced.clreplaced);
    } else {
        conf.setMapperClreplaced(MRSharedCaching.MapClreplaced2.clreplaced);
    }
    // Turn on sharing
    conf.set("mapred.cache.shared.enabled", "true");
    return fs;
}

8 View Complete Implementation : MRSharedCaching.java
Copyright Apache License 2.0
Author : facebookarchive
// Boilerplate code
public static FileSystem setupJob(String indir, String outdir, String cacheDir, JobConf conf, String input, boolean withSymlink) throws IOException {
    final Path inDir = new Path(indir);
    final Path outDir = new Path(outdir);
    FileSystem fs = FileSystem.get(conf);
    fs.delete(outDir, true);
    if (!fs.mkdirs(inDir)) {
        throw new IOException("Mkdirs failed to create " + inDir.toString());
    }
    {
        DataOutputStream file = fs.create(new Path(inDir, "part-0"));
        file.writeBytes(input);
        file.close();
    }
    conf.setJobName("sharedcachetest");
    // the keys are words (strings)
    conf.setOutputKeyClreplaced(Text.clreplaced);
    // the values are counts (ints)
    conf.setOutputValueClreplaced(IntWritable.clreplaced);
    conf.setCombinerClreplaced(MRSharedCaching.ReduceClreplaced.clreplaced);
    conf.setReducerClreplaced(MRSharedCaching.ReduceClreplaced.clreplaced);
    FileInputFormat.setInputPaths(conf, inDir);
    FileOutputFormat.setOutputPath(conf, outDir);
    conf.setNumMapTasks(1);
    conf.setNumReduceTasks(1);
    conf.setSpeculativeExecution(false);
    if (!withSymlink) {
        conf.setMapperClreplaced(MRSharedCaching.MapClreplaced.clreplaced);
    } else {
        conf.setMapperClreplaced(MRSharedCaching.MapClreplaced2.clreplaced);
    }
    // Turn on sharing
    conf.set("mapred.cache.shared.enabled", "true");
    return fs;
}

7 View Complete Implementation : SleepJob.java
Copyright Apache License 2.0
Author : madiator
public JobConf setupJobConf(int numMapper, int numReducer, long mapSleepTime, int mapSleepCount, long reduceSleepTime, int reduceSleepCount, boolean doSpeculation, List<String> slowMaps, List<String> slowReduces, int slowRatio, int countersPerTask, List<String> hosts, int hostsPerSplit, boolean setup) {
    JobConf job = new JobConf(getConf(), SleepJob.clreplaced);
    job.setNumMapTasks(numMapper);
    job.setNumReduceTasks(numReducer);
    job.setMapperClreplaced(SleepJob.clreplaced);
    job.setMapOutputKeyClreplaced(IntWritable.clreplaced);
    job.setMapOutputValueClreplaced(NullWritable.clreplaced);
    job.setReducerClreplaced(SleepJob.clreplaced);
    job.setOutputFormat(NullOutputFormat.clreplaced);
    job.setJobSetupCleanupNeeded(setup);
    job.setInputFormat(SleepInputFormat.clreplaced);
    job.setParreplacedionerClreplaced(SleepJob.clreplaced);
    job.setJobName("Sleep job");
    FileInputFormat.addInputPath(job, new Path("ignored"));
    job.setLong("sleep.job.map.sleep.time", mapSleepTime);
    job.setLong("sleep.job.reduce.sleep.time", reduceSleepTime);
    job.setInt("sleep.job.map.sleep.count", mapSleepCount);
    job.setInt("sleep.job.reduce.sleep.count", reduceSleepCount);
    job.setSpeculativeExecution(doSpeculation);
    job.setInt(SLOW_RATIO, slowRatio);
    job.setStrings(SLOW_MAPS, slowMaps.toArray(new String[slowMaps.size()]));
    job.setStrings(SLOW_REDUCES, slowMaps.toArray(new String[slowReduces.size()]));
    job.setInt("sleep.job.counters.per.task", countersPerTask);
    job.setStrings(HOSTS_FOR_LOCALITY, hosts.toArray(new String[hosts.size()]));
    job.setInt(HOSTS_PER_SPLIT, hostsPerSplit);
    return job;
}

7 View Complete Implementation : DBCountPageView.java
Copyright Apache License 2.0
Author : madiator
@Override
public // Usage DBCountPageView [driverClreplaced dburl]
int run(String[] args) throws Exception {
    String driverClreplacedName = DRIVER_CLreplaced;
    String url = DB_URL;
    if (args.length > 1) {
        driverClreplacedName = args[0];
        url = args[1];
    }
    initialize(driverClreplacedName, url);
    JobConf job = new JobConf(getConf(), DBCountPageView.clreplaced);
    job.setJobName("Count Pageviews of URLs");
    job.setMapperClreplaced(PageviewMapper.clreplaced);
    job.setCombinerClreplaced(LongSumReducer.clreplaced);
    job.setReducerClreplaced(PageviewReducer.clreplaced);
    DBConfiguration.configureDB(job, driverClreplacedName, url);
    DBInputFormat.setInput(job, AccessRecord.clreplaced, "Access", null, "url", AccessFieldNames);
    DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);
    job.setMapOutputKeyClreplaced(Text.clreplaced);
    job.setMapOutputValueClreplaced(LongWritable.clreplaced);
    job.setOutputKeyClreplaced(PageviewRecord.clreplaced);
    job.setOutputValueClreplaced(NullWritable.clreplaced);
    try {
        JobClient.runJob(job);
        boolean correct = verify();
        if (!correct) {
            throw new RuntimeException("Evaluation was not correct!");
        }
    } finally {
        shutdown();
    }
    return 0;
}

7 View Complete Implementation : WordCount.java
Copyright Apache License 2.0
Author : facebookarchive
/**
 * The main driver for word count map/reduce program.
 * Invoke this method to submit the map/reduce job.
 * @throws IOException When there is communication problems with the
 *                     job tracker.
 */
public int run(String[] args) throws Exception {
    JobConf conf = new JobConf(getConf(), WordCount.clreplaced);
    conf.setJobName("wordcount");
    // the keys are words (strings)
    conf.setOutputKeyClreplaced(Text.clreplaced);
    // the values are counts (ints)
    conf.setOutputValueClreplaced(IntWritable.clreplaced);
    conf.setMapperClreplaced(MapClreplaced.clreplaced);
    conf.setCombinerClreplaced(Reduce.clreplaced);
    conf.setReducerClreplaced(Reduce.clreplaced);
    List<String> other_args = new ArrayList<String>();
    for (int i = 0; i < args.length; ++i) {
        try {
            if ("-m".equals(args[i])) {
                conf.setNumMapTasks(Integer.parseInt(args[++i]));
            } else if ("-r".equals(args[i])) {
                conf.setNumReduceTasks(Integer.parseInt(args[++i]));
            } else {
                other_args.add(args[i]);
            }
        } catch (NumberFormatException except) {
            System.out.println("ERROR: Integer expected instead of " + args[i]);
            return printUsage();
        } catch (ArrayIndexOutOfBoundsException except) {
            System.out.println("ERROR: Required parameter missing from " + args[i - 1]);
            return printUsage();
        }
    }
    // Make sure there are exactly 2 parameters left.
    if (other_args.size() != 2) {
        System.out.println("ERROR: Wrong number of parameters: " + other_args.size() + " instead of 2.");
        return printUsage();
    }
    FileInputFormat.setInputPaths(conf, other_args.get(0));
    FileOutputFormat.setOutputPath(conf, new Path(other_args.get(1)));
    JobClient.runJob(conf);
    return 0;
}

7 View Complete Implementation : DBCountPageView.java
Copyright Apache License 2.0
Author : facebookarchive
@Override
public // Usage DBCountPageView [driverClreplaced dburl]
int run(String[] args) throws Exception {
    String driverClreplacedName = DRIVER_CLreplaced;
    String url = DB_URL;
    if (args.length > 1) {
        driverClreplacedName = args[0];
        url = args[1];
    }
    initialize(driverClreplacedName, url);
    JobConf job = new JobConf(getConf(), DBCountPageView.clreplaced);
    job.setJobName("Count Pageviews of URLs");
    job.setMapperClreplaced(PageviewMapper.clreplaced);
    job.setCombinerClreplaced(LongSumReducer.clreplaced);
    job.setReducerClreplaced(PageviewReducer.clreplaced);
    DBConfiguration.configureDB(job, driverClreplacedName, url);
    DBInputFormat.setInput(job, AccessRecord.clreplaced, "Access", null, "url", AccessFieldNames);
    DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);
    job.setMapOutputKeyClreplaced(Text.clreplaced);
    job.setMapOutputValueClreplaced(LongWritable.clreplaced);
    job.setOutputKeyClreplaced(PageviewRecord.clreplaced);
    job.setOutputValueClreplaced(NullWritable.clreplaced);
    try {
        JobClient.runJob(job);
        boolean correct = verify();
        if (!correct) {
            throw new RuntimeException("Evaluation was not correct!");
        }
    } finally {
        shutdown();
    }
    return 0;
}

7 View Complete Implementation : DBCountPageView.java
Copyright Apache License 2.0
Author : iVCE
@Override
public // Usage DBCountPageView [driverClreplaced dburl]
int run(String[] args) throws Exception {
    String driverClreplacedName = DRIVER_CLreplaced;
    String url = DB_URL;
    if (args.length > 1) {
        driverClreplacedName = args[0];
        url = args[1];
    }
    initialize(driverClreplacedName, url);
    JobConf job = new JobConf(getConf(), DBCountPageView.clreplaced);
    job.setJobName("Count Pageviews of URLs");
    job.setMapperClreplaced(PageviewMapper.clreplaced);
    job.setCombinerClreplaced(LongSumReducer.clreplaced);
    job.setReducerClreplaced(PageviewReducer.clreplaced);
    DBConfiguration.configureDB(job, driverClreplacedName, url);
    DBInputFormat.setInput(job, AccessRecord.clreplaced, "Access", null, "url", AccessFieldNames);
    DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);
    job.setMapOutputKeyClreplaced(Text.clreplaced);
    job.setMapOutputValueClreplaced(LongWritable.clreplaced);
    job.setOutputKeyClreplaced(PageviewRecord.clreplaced);
    job.setOutputValueClreplaced(NullWritable.clreplaced);
    try {
        JobClient.runJob(job);
        boolean correct = verify();
        if (!correct) {
            throw new RuntimeException("Evaluation was not correct!");
        }
    } finally {
        shutdown();
    }
    return 0;
}

7 View Complete Implementation : SleepJob.java
Copyright Apache License 2.0
Author : iVCE
public JobConf setupJobConf(int numMapper, int numReducer, long mapSleepTime, int mapSleepCount, long reduceSleepTime, int reduceSleepCount, boolean doSpeculation, List<String> slowMaps, List<String> slowReduces, int slowRatio, int countersPerTask, List<String> hosts, int hostsPerSplit, boolean setup) {
    JobConf job = new JobConf(getConf(), SleepJob.clreplaced);
    job.setNumMapTasks(numMapper);
    job.setNumReduceTasks(numReducer);
    job.setMapperClreplaced(SleepJob.clreplaced);
    job.setMapOutputKeyClreplaced(IntWritable.clreplaced);
    job.setMapOutputValueClreplaced(NullWritable.clreplaced);
    job.setReducerClreplaced(SleepJob.clreplaced);
    job.setOutputFormat(NullOutputFormat.clreplaced);
    job.setJobSetupCleanupNeeded(setup);
    job.setInputFormat(SleepInputFormat.clreplaced);
    job.setParreplacedionerClreplaced(SleepJob.clreplaced);
    job.setJobName("Sleep job");
    FileInputFormat.addInputPath(job, new Path("ignored"));
    job.setLong("sleep.job.map.sleep.time", mapSleepTime);
    job.setLong("sleep.job.reduce.sleep.time", reduceSleepTime);
    job.setInt("sleep.job.map.sleep.count", mapSleepCount);
    job.setInt("sleep.job.reduce.sleep.count", reduceSleepCount);
    job.setSpeculativeExecution(doSpeculation);
    job.setInt(SLOW_RATIO, slowRatio);
    job.setStrings(SLOW_MAPS, slowMaps.toArray(new String[slowMaps.size()]));
    job.setStrings(SLOW_REDUCES, slowMaps.toArray(new String[slowReduces.size()]));
    job.setInt("sleep.job.counters.per.task", countersPerTask);
    job.setStrings(HOSTS_FOR_LOCALITY, hosts.toArray(new String[hosts.size()]));
    job.setInt(HOSTS_PER_SPLIT, hostsPerSplit);
    return job;
}