org.apache.flink.api.common.JobID - java examples

Here are the examples of the java api org.apache.flink.api.common.JobID taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

155 Examples 7

19 View Complete Implementation : JobManagerGroupTest.java
Copyright Apache License 2.0
Author : apache
// ------------------------------------------------------------------------
// adding and removing jobs
// ------------------------------------------------------------------------
@Test
public void addAndRemoveJobs() throws Exception {
    MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.defaultMetricRegistryConfiguration());
    final JobManagerMetricGroup group = new JobManagerMetricGroup(registry, "localhost");
    final JobID jid1 = new JobID();
    final JobID jid2 = new JobID();
    final String jobName1 = "testjob";
    final String jobName2 = "anotherJob";
    JobManagerJobMetricGroup jmJobGroup11 = group.addJob(new JobGraph(jid1, jobName1));
    JobManagerJobMetricGroup jmJobGroup12 = group.addJob(new JobGraph(jid1, jobName1));
    JobManagerJobMetricGroup jmJobGroup21 = group.addJob(new JobGraph(jid2, jobName2));
    replacedertEquals(jmJobGroup11, jmJobGroup12);
    replacedertEquals(2, group.numRegisteredJobMetricGroups());
    group.removeJob(jid1);
    replacedertTrue(jmJobGroup11.isClosed());
    replacedertEquals(1, group.numRegisteredJobMetricGroups());
    group.removeJob(jid2);
    replacedertTrue(jmJobGroup21.isClosed());
    replacedertEquals(0, group.numRegisteredJobMetricGroups());
    registry.shutdown().get();
}

19 View Complete Implementation : TaskManagerMetricGroup.java
Copyright Apache License 2.0
Author : apache
public void removeJobMetricsGroup(JobID jobId, TaskManagerJobMetricGroup group) {
    if (jobId == null || group == null || !group.isClosed()) {
        return;
    }
    synchronized (this) {
        // optimistically remove the currently contained group, and check later if it was correct
        TaskManagerJobMetricGroup containedGroup = jobs.remove(jobId);
        // check if another group was actually contained, and restore that one
        if (containedGroup != null && containedGroup != group) {
            jobs.put(jobId, containedGroup);
        }
    }
}

19 View Complete Implementation : CheckpointCoordinatorDeActivator.java
Copyright Apache License 2.0
Author : apache
@Override
public void jobStatusChanges(JobID jobId, JobStatus newJobStatus, long timestamp, Throwable error) {
    if (newJobStatus == JobStatus.RUNNING) {
        // start the checkpoint scheduler
        coordinator.startCheckpointScheduler();
    } else {
        // anything else should stop the trigger for now
        coordinator.stopCheckpointScheduler();
    }
}

19 View Complete Implementation : PermanentBlobCache.java
Copyright Apache License 2.0
Author : apache
/**
 * Returns the path to a local copy of the file replacedociated with the provided job ID and blob
 * key.
 *
 * <p>We will first attempt to serve the BLOB from the local storage. If the BLOB is not in
 * there, we will try to download it from the HA store, or directly from the {@link BlobServer}.
 *
 * @param jobId
 * 		ID of the job this blob belongs to
 * @param key
 * 		blob key replacedociated with the requested file
 *
 * @return The path to the file.
 *
 * @throws java.io.FileNotFoundException
 * 		if the BLOB does not exist;
 * @throws IOException
 * 		if any other error occurs when retrieving the file
 */
@Override
public File getFile(JobID jobId, PermanentBlobKey key) throws IOException {
    checkNotNull(jobId);
    return getFileInternal(jobId, key);
}

19 View Complete Implementation : JarRunResponseBody.java
Copyright Apache License 2.0
Author : apache
/**
 * Response for {@link JarRunHandler}.
 */
public clreplaced JarRunResponseBody implements ResponseBody {

    @JsonProperty("jobid")
    @JsonDeserialize(using = JobIDDeserializer.clreplaced)
    @JsonSerialize(using = JobIDSerializer.clreplaced)
    private final JobID jobId;

    @JsonCreator
    public JarRunResponseBody(@JsonProperty("jobid") @JsonDeserialize(using = JobIDDeserializer.clreplaced) final JobID jobId) {
        this.jobId = requireNonNull(jobId);
    }

    public JobID getJobId() {
        return jobId;
    }
}

19 View Complete Implementation : BlobCacheRetriesTest.java
Copyright Apache License 2.0
Author : apache
/**
 * A test where the BlobCache must use the BlobServer and the connection
 * fails too often which eventually fails the GET request.
 *
 * @param config
 * 		configuration to use (the BlobCache will get some additional settings
 * 		set compared to this one)
 * @param blobType
 * 		whether the BLOB should become permanent or transient
 */
private static void testBlobFetchWithTooManyFailures(final Configuration config, final BlobStore blobStore, @Nullable final JobID jobId, BlobKey.BlobType blobType) throws IOException {
    final byte[] data = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 0 };
    try (BlobServer server = new TestingFailingBlobServer(config, blobStore, 0, 10);
        BlobCacheService cache = new BlobCacheService(config, new VoidBlobStore(), new InetSocketAddress("localhost", server.getPort()))) {
        server.start();
        // upload some blob
        final BlobKey key = put(server, jobId, data, blobType);
        // trigger a download - it should fail eventually
        try {
            verifyContents(cache, jobId, key, data);
            fail("This should fail");
        } catch (IOException e) {
        // as we expected
        }
    }
}

19 View Complete Implementation : CliFrontend.java
Copyright Apache License 2.0
Author : apache
// --------------------------------------------------------------------------------------------
// Internal methods
// --------------------------------------------------------------------------------------------
private JobID parseJobId(String jobIdString) throws CliArgsException {
    if (jobIdString == null) {
        throw new CliArgsException("Missing JobId");
    }
    final JobID jobId;
    try {
        jobId = JobID.fromHexString(jobIdString);
    } catch (IllegalArgumentException e) {
        throw new CliArgsException(e.getMessage());
    }
    return jobId;
}

19 View Complete Implementation : PermanentBlobCache.java
Copyright Apache License 2.0
Author : apache
public int getNumberOfReferenceHolders(JobID jobId) {
    checkNotNull(jobId);
    synchronized (jobRefCounters) {
        RefCount ref = jobRefCounters.get(jobId);
        if (ref == null) {
            return 0;
        } else {
            return ref.references;
        }
    }
}

19 View Complete Implementation : JobExecutionException.java
Copyright Apache License 2.0
Author : apache
/**
 * This exception is the base exception for all exceptions that denote any failure during
 * the execution of a job.
 */
public clreplaced JobExecutionException extends FlinkException {

    private static final long serialVersionUID = 2818087325120827525L;

    private final JobID jobID;

    /**
     * Constructs a new job execution exception.
     *
     * @param jobID The job's ID.
     * @param msg The cause for the execution exception.
     * @param cause The cause of the exception
     */
    public JobExecutionException(JobID jobID, String msg, Throwable cause) {
        super(msg, cause);
        this.jobID = jobID;
    }

    public JobExecutionException(JobID jobID, String msg) {
        super(msg);
        this.jobID = jobID;
    }

    public JobExecutionException(JobID jobID, Throwable cause) {
        super(cause);
        this.jobID = jobID;
    }

    public JobID getJobID() {
        return jobID;
    }
}

19 View Complete Implementation : KvStateRegistry.java
Copyright Apache License 2.0
Author : apache
// ------------------------------------------------------------------------
// Internal methods
// ------------------------------------------------------------------------
private KvStateRegistryListener getKvStateRegistryListener(JobID jobId) {
    // first check whether we are running the legacy code which registers
    // a single listener under HighAvailabilityServices.DEFAULT_JOB_ID
    KvStateRegistryListener listener = listeners.get(HighAvailabilityServices.DEFAULT_JOB_ID);
    if (listener == null) {
        listener = listeners.get(jobId);
    }
    return listener;
}

19 View Complete Implementation : StreamIterationHead.java
Copyright Apache License 2.0
Author : apache
// ------------------------------------------------------------------------
// Utilities
// ------------------------------------------------------------------------
/**
 * Creates the identification string with which head and tail task find the shared blocking
 * queue for the back channel. The identification string is unique per parallel head/tail pair
 * per iteration per job.
 *
 * @param jid The job ID.
 * @param iterationID The id of the iteration in the job.
 * @param subtaskIndex The parallel subtask number
 * @return The identification string.
 */
public static String createBrokerIdString(JobID jid, String iterationID, int subtaskIndex) {
    return jid + "-" + iterationID + "-" + subtaskIndex;
}

19 View Complete Implementation : SlotOccupiedException.java
Copyright Apache License 2.0
Author : apache
/**
 * Exception which signals that a slot is already occupied by the given
 * {@link AllocationID}.
 */
public clreplaced SlotOccupiedException extends SlotAllocationException {

    private static final long serialVersionUID = -3986333914244338888L;

    private final AllocationID allocationId;

    private final JobID jobId;

    public SlotOccupiedException(String message, AllocationID allocationId, JobID jobId) {
        super(message);
        this.allocationId = Preconditions.checkNotNull(allocationId);
        this.jobId = Preconditions.checkNotNull(jobId);
    }

    public AllocationID getAllocationId() {
        return allocationId;
    }

    public JobID getJobId() {
        return jobId;
    }
}

19 View Complete Implementation : StandaloneCompletedCheckpointStoreTest.java
Copyright Apache License 2.0
Author : apache
@Test
public void testPreferCheckpointWithSavepoint() throws Exception {
    StandaloneCompletedCheckpointStore store = new StandaloneCompletedCheckpointStore(5);
    JobID jobId = new JobID();
    store.addCheckpoint(checkpoint(jobId, 1L));
    store.addCheckpoint(savepoint(jobId, 2L));
    store.addCheckpoint(savepoint(jobId, 3L));
    CompletedCheckpoint latestCheckpoint = store.getLatestCheckpoint(true);
    replacedertThat(latestCheckpoint.getCheckpointID(), equalTo(1L));
}

19 View Complete Implementation : BlobLibraryCacheManager.java
Copyright Apache License 2.0
Author : apache
@Override
public ClreplacedLoader getClreplacedLoader(JobID jobId) {
    checkNotNull(jobId, "The JobId must not be null.");
    synchronized (lockObject) {
        LibraryCacheEntry entry = cacheEntries.get(jobId);
        if (entry == null) {
            throw new IllegalStateException("No libraries are registered for job " + jobId);
        }
        return entry.getClreplacedLoader();
    }
}

19 View Complete Implementation : BlobServerPutTest.java
Copyright Apache License 2.0
Author : apache
// --------------------------------------------------------------------------------------------
/**
 * Helper to choose the right {@link BlobServer#putTransient} method.
 *
 * @param blobType
 * 		whether the BLOB should become permanent or transient
 *
 * @return blob key for the uploaded data
 */
static BlobKey put(BlobService service, @Nullable JobID jobId, InputStream data, BlobKey.BlobType blobType) throws IOException {
    if (blobType == PERMANENT_BLOB) {
        if (service instanceof BlobServer) {
            return ((BlobServer) service).putPermanent(jobId, data);
        } else {
            throw new UnsupportedOperationException("uploading streams is only possible at the BlobServer");
        }
    } else if (jobId == null) {
        return service.getTransientBlobService().putTransient(data);
    } else {
        return service.getTransientBlobService().putTransient(jobId, data);
    }
}

19 View Complete Implementation : KvStateClientProxyImplTest.java
Copyright Apache License 2.0
Author : apache
/**
 * Tests that we can set and retrieve the {@link KvStateLocationOracle}.
 */
@Test
public void testKvStateLocationOracle() {
    final JobID jobId1 = new JobID();
    final TestingKvStateLocationOracle kvStateLocationOracle1 = new TestingKvStateLocationOracle();
    kvStateClientProxy.updateKvStateLocationOracle(jobId1, kvStateLocationOracle1);
    final JobID jobId2 = new JobID();
    final TestingKvStateLocationOracle kvStateLocationOracle2 = new TestingKvStateLocationOracle();
    kvStateClientProxy.updateKvStateLocationOracle(jobId2, kvStateLocationOracle2);
    replacedertThat(kvStateClientProxy.getKvStateLocationOracle(new JobID()), nullValue());
    replacedertThat(kvStateClientProxy.getKvStateLocationOracle(jobId1), equalTo(kvStateLocationOracle1));
    replacedertThat(kvStateClientProxy.getKvStateLocationOracle(jobId2), equalTo(kvStateLocationOracle2));
    kvStateClientProxy.updateKvStateLocationOracle(jobId1, null);
    replacedertThat(kvStateClientProxy.getKvStateLocationOracle(jobId1), nullValue());
}

19 View Complete Implementation : AllocatedSlotReport.java
Copyright Apache License 2.0
Author : apache
/**
 * The report of currently allocated slots from a given TaskExecutor by a JobMaster.
 * This report is sent periodically to the TaskExecutor in order to reconcile the internal state of slot allocations.
 */
public clreplaced AllocatedSlotReport implements Serializable {

    private static final long serialVersionUID = 1L;

    private final JobID jobId;

    /**
     * The allocated slots in slot pool.
     */
    private final Collection<AllocatedSlotInfo> allocatedSlotInfos;

    public AllocatedSlotReport(JobID jobId, Collection<AllocatedSlotInfo> allocatedSlotInfos) {
        this.jobId = checkNotNull(jobId);
        this.allocatedSlotInfos = checkNotNull(allocatedSlotInfos);
    }

    public JobID getJobId() {
        return jobId;
    }

    public Collection<AllocatedSlotInfo> getAllocatedSlotInfos() {
        return Collections.unmodifiableCollection(allocatedSlotInfos);
    }

    @Override
    public String toString() {
        return "AllocatedSlotReport{" + "jobId=" + jobId + ", allocatedSlotInfos=" + allocatedSlotInfos + '}';
    }
}

19 View Complete Implementation : WebMonitorMessagesTest.java
Copyright Apache License 2.0
Author : apache
@Test
public void testJobDetailsMessage() {
    try {
        final Random rnd = new Random();
        int[] numVerticesPerState = new int[ExecutionState.values().length];
        int numTotal = 0;
        for (int i = 0; i < numVerticesPerState.length; i++) {
            int count = rnd.nextInt(55);
            numVerticesPerState[i] = count;
            numTotal += count;
        }
        long time = rnd.nextLong();
        long endTime = rnd.nextBoolean() ? -1L : time + rnd.nextInt();
        long lastModified = endTime == -1 ? time + rnd.nextInt() : endTime;
        String name = GenericMessageTester.randomString(rnd);
        JobID jid = GenericMessageTester.randomJobId(rnd);
        JobStatus status = GenericMessageTester.randomJobStatus(rnd);
        JobDetails msg1 = new JobDetails(jid, name, time, endTime, endTime - time, status, lastModified, numVerticesPerState, numTotal);
        JobDetails msg2 = new JobDetails(jid, name, time, endTime, endTime - time, status, lastModified, numVerticesPerState, numTotal);
        GenericMessageTester.testMessageInstances(msg1, msg2);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

19 View Complete Implementation : KvStateRegistry.java
Copyright Apache License 2.0
Author : apache
/**
 * Registers a listener with the registry.
 *
 * @param jobId identifying the job for which to register a {@link KvStateRegistryListener}
 * @param listener The registry listener.
 * @throws IllegalStateException If there is a registered listener
 */
public void registerListener(JobID jobId, KvStateRegistryListener listener) {
    final KvStateRegistryListener previousValue = listeners.putIfAbsent(jobId, listener);
    if (previousValue != null) {
        throw new IllegalStateException("Listener already registered under " + jobId + '.');
    }
}

19 View Complete Implementation : AsynchronousJobOperationKey.java
Copyright Apache License 2.0
Author : apache
/**
 * A pair of {@link JobID} and {@link TriggerId} used as a key to a hash based
 * collection.
 *
 * @see AbstractAsynchronousOperationHandlers
 */
@Immutable
public clreplaced AsynchronousJobOperationKey extends OperationKey {

    private final JobID jobId;

    private AsynchronousJobOperationKey(final TriggerId triggerId, final JobID jobId) {
        super(triggerId);
        this.jobId = requireNonNull(jobId);
    }

    public static AsynchronousJobOperationKey of(final TriggerId triggerId, final JobID jobId) {
        return new AsynchronousJobOperationKey(triggerId, jobId);
    }

    @Override
    public boolean equals(Object o) {
        if (this == o) {
            return true;
        }
        if (o == null || getClreplaced() != o.getClreplaced()) {
            return false;
        }
        if (!super.equals(o)) {
            return false;
        }
        AsynchronousJobOperationKey that = (AsynchronousJobOperationKey) o;
        return Objects.equals(jobId, that.jobId);
    }

    @Override
    public int hashCode() {
        return Objects.hash(super.hashCode(), jobId);
    }
}

19 View Complete Implementation : JobIdTest.java
Copyright Apache License 2.0
Author : apache
@Test
public void testConvertToByteBuffer() {
    try {
        JobID origID = new JobID();
        byte[] bytes = origID.getBytes();
        ByteBuffer buffer = ByteBuffer.wrap(bytes);
        JobID copy1 = JobID.fromByteBuffer(buffer);
        JobID copy2 = JobID.fromByteArray(bytes);
        replacedertEquals(origID, copy1);
        replacedertEquals(origID, copy2);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

19 View Complete Implementation : AggregatingJobsMetricsHandlerTest.java
Copyright Apache License 2.0
Author : apache
/**
 * Tests for the {@link AggregatingJobsMetricsHandler}.
 */
public clreplaced AggregatingJobsMetricsHandlerTest extends AggregatingMetricsHandlerTestBase<AggregatingJobsMetricsHandler, AggregatedJobMetricsParameters> {

    private static final JobID JOB_ID_1 = JobID.generate();

    private static final JobID JOB_ID_2 = JobID.generate();

    private static final JobID JOB_ID_3 = JobID.generate();

    @Override
    protected Tuple2<String, List<String>> getFilter() {
        return Tuple2.of("jobs", Arrays.asList(JOB_ID_1.toString(), JOB_ID_3.toString()));
    }

    @Override
    protected Collection<MetricDump> getMetricDumps() {
        Collection<MetricDump> dumps = new ArrayList<>(3);
        QueryScopeInfo.JobQueryScopeInfo job = new QueryScopeInfo.JobQueryScopeInfo(JOB_ID_1.toString(), "abc");
        MetricDump.CounterDump cd1 = new MetricDump.CounterDump(job, "metric1", 1);
        dumps.add(cd1);
        QueryScopeInfo.JobQueryScopeInfo job2 = new QueryScopeInfo.JobQueryScopeInfo(JOB_ID_2.toString(), "abc");
        MetricDump.CounterDump cd2 = new MetricDump.CounterDump(job2, "metric1", 3);
        dumps.add(cd2);
        QueryScopeInfo.JobQueryScopeInfo job3 = new QueryScopeInfo.JobQueryScopeInfo(JOB_ID_3.toString(), "abc");
        MetricDump.CounterDump cd3 = new MetricDump.CounterDump(job3, "metric2", 5);
        dumps.add(cd3);
        return dumps;
    }

    @Override
    protected AggregatingJobsMetricsHandler getHandler(GatewayRetriever<? extends RestfulGateway> leaderRetriever, Time timeout, Map<String, String> responseHeaders, Executor executor, MetricFetcher fetcher) {
        return new AggregatingJobsMetricsHandler(leaderRetriever, timeout, responseHeaders, executor, fetcher);
    }
}

19 View Complete Implementation : TransientBlobCache.java
Copyright Apache License 2.0
Author : apache
/**
 * Returns a file handle to the file replacedociated with the given blob key on the blob
 * server.
 *
 * @param jobId
 * 		ID of the job this blob belongs to (or <tt>null</tt> if job-unrelated)
 * @param key
 * 		identifying the file
 *
 * @return file handle to the file
 *
 * @throws IOException
 * 		if creating the directory fails
 */
@VisibleForTesting
public File getStorageLocation(@Nullable JobID jobId, BlobKey key) throws IOException {
    return BlobUtils.getStorageLocation(storageDir, jobId, key);
}

19 View Complete Implementation : ZooKeeperRunningJobsRegistry.java
Copyright Apache License 2.0
Author : apache
@Override
public void clearJob(JobID jobID) throws IOException {
    checkNotNull(jobID);
    try {
        final String zkPath = createZkPath(jobID);
        this.client.newNamespaceAwareEnsurePath(zkPath).ensure(client.getZookeeperClient());
        this.client.delete().forPath(zkPath);
    } catch (Exception e) {
        throw new IOException("Failed to clear job state from ZooKeeper for job " + jobID, e);
    }
}

19 View Complete Implementation : BlobCachePutTest.java
Copyright Apache License 2.0
Author : apache
/**
 * Checks that the given blob will be deleted at the {@link BlobServer} eventually (waits at most 30s).
 *
 * @param server
 * 		BLOB server
 * @param jobId
 * 		job ID or <tt>null</tt> if job-unrelated
 * @param keys
 * 		key(s) identifying the BLOB to request
 */
static void verifyDeletedEventually(BlobServer server, @Nullable JobID jobId, BlobKey... keys) throws IOException, InterruptedException {
    long deadline = System.currentTimeMillis() + 30_000L;
    do {
        Thread.sleep(10);
    } while (checkFilesExist(jobId, Arrays.asList(keys), server, false) != 0 && System.currentTimeMillis() < deadline);
    for (BlobKey key : keys) {
        verifyDeleted(server, jobId, key);
    }
}

19 View Complete Implementation : BlobCacheRetriesTest.java
Copyright Apache License 2.0
Author : apache
/**
 * A test where the BlobCache must use the BlobServer and the connection
 * fails twice and then the get operation succeeds.
 *
 * @param config
 * 		configuration to use (the BlobCache will get some additional settings
 * 		set compared to this one)
 * @param blobType
 * 		whether the BLOB should become permanent or transient
 */
private static void testBlobFetchRetries(final Configuration config, final BlobStore blobStore, @Nullable final JobID jobId, BlobKey.BlobType blobType) throws IOException {
    final byte[] data = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 0 };
    try (BlobServer server = new TestingFailingBlobServer(config, blobStore, 2);
        BlobCacheService cache = new BlobCacheService(config, new VoidBlobStore(), new InetSocketAddress("localhost", server.getPort()))) {
        server.start();
        // upload some blob
        final BlobKey key = put(server, jobId, data, blobType);
        // trigger a download - it should fail the first two times, but retry, and succeed eventually
        verifyContents(cache, jobId, key, data);
    }
}

19 View Complete Implementation : PermanentBlobCache.java
Copyright Apache License 2.0
Author : apache
/**
 * Returns a file handle to the file replacedociated with the given blob key on the blob
 * server.
 *
 * @param jobId
 * 		ID of the job this blob belongs to (or <tt>null</tt> if job-unrelated)
 * @param key
 * 		identifying the file
 *
 * @return file handle to the file
 *
 * @throws IOException
 * 		if creating the directory fails
 */
@VisibleForTesting
public File getStorageLocation(JobID jobId, BlobKey key) throws IOException {
    checkNotNull(jobId);
    return BlobUtils.getStorageLocation(storageDir, jobId, key);
}

19 View Complete Implementation : MiniClusterClient.java
Copyright Apache License 2.0
Author : apache
@Override
public CompletableFuture<Map<String, Object>> getAcreplacedulators(JobID jobID, ClreplacedLoader loader) {
    return miniCluster.getExecutionGraph(jobID).thenApply(AccessExecutionGraph::getAcreplacedulatorsSerialized).thenApply(acreplacedulators -> {
        try {
            return AcreplacedulatorHelper.deserializeAndUnwrapAcreplacedulators(acreplacedulators, loader);
        } catch (Exception e) {
            throw new CompletionException("Cannot deserialize and unwrap acreplacedulators properly.", e);
        }
    });
}

19 View Complete Implementation : PermanentBlobCache.java
Copyright Apache License 2.0
Author : apache
/**
 * Unregisters use of job-related BLOBs and allow them to be released.
 *
 * @param jobId
 * 		ID of the job this blob belongs to
 *
 * @see #registerJob(JobID)
 */
public void releaseJob(JobID jobId) {
    checkNotNull(jobId);
    synchronized (jobRefCounters) {
        RefCount ref = jobRefCounters.get(jobId);
        if (ref == null || ref.references == 0) {
            log.warn("improper use of releaseJob() without a matching number of registerJob() calls for jobId " + jobId);
            return;
        }
        --ref.references;
        if (ref.references == 0) {
            ref.keepUntil = System.currentTimeMillis() + cleanupInterval;
        }
    }
}

19 View Complete Implementation : JobManagerMetricGroup.java
Copyright Apache License 2.0
Author : apache
// ------------------------------------------------------------------------
// job groups
// ------------------------------------------------------------------------
public JobManagerJobMetricGroup addJob(JobGraph job) {
    JobID jobId = job.getJobID();
    String jobName = job.getName();
    // get or create a jobs metric group
    JobManagerJobMetricGroup currentJobGroup;
    synchronized (this) {
        if (!isClosed()) {
            currentJobGroup = jobs.get(jobId);
            if (currentJobGroup == null || currentJobGroup.isClosed()) {
                currentJobGroup = new JobManagerJobMetricGroup(registry, this, jobId, jobName);
                jobs.put(jobId, currentJobGroup);
            }
            return currentJobGroup;
        } else {
            return null;
        }
    }
}

19 View Complete Implementation : BlobLibraryCacheManager.java
Copyright Apache License 2.0
Author : apache
/**
 * Gets the number of tasks holding {@link ClreplacedLoader} references for the given job.
 *
 * @param jobId ID of a job
 *
 * @return number of reference holders
 */
int getNumberOfReferenceHolders(JobID jobId) {
    synchronized (lockObject) {
        LibraryCacheEntry entry = cacheEntries.get(jobId);
        return entry == null ? 0 : entry.getNumberOfReferenceHolders();
    }
}

19 View Complete Implementation : JobManagerGroupTest.java
Copyright Apache License 2.0
Author : apache
@Test
public void testCloseClosesAll() throws Exception {
    MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.defaultMetricRegistryConfiguration());
    final JobManagerMetricGroup group = new JobManagerMetricGroup(registry, "localhost");
    final JobID jid1 = new JobID();
    final JobID jid2 = new JobID();
    final String jobName1 = "testjob";
    final String jobName2 = "anotherJob";
    JobManagerJobMetricGroup jmJobGroup11 = group.addJob(new JobGraph(jid1, jobName1));
    JobManagerJobMetricGroup jmJobGroup21 = group.addJob(new JobGraph(jid2, jobName2));
    group.close();
    replacedertTrue(jmJobGroup11.isClosed());
    replacedertTrue(jmJobGroup21.isClosed());
    registry.shutdown().get();
}

19 View Complete Implementation : AbstractFsCheckpointStorage.java
Copyright Apache License 2.0
Author : apache
// ------------------------------------------------------------------------
// Creating and resolving paths
// ------------------------------------------------------------------------
/**
 * Builds directory into which a specific job checkpoints, meaning the directory inside which
 * it creates the checkpoint-specific subdirectories.
 *
 * <p>This method only succeeds if a base checkpoint directory has been set; otherwise
 * the method fails with an exception.
 *
 * @param jobId The ID of the job
 * @return The job's checkpoint directory, re
 *
 * @throws UnsupportedOperationException Thrown, if no base checkpoint directory has been set.
 */
protected static Path getCheckpointDirectoryForJob(Path baseCheckpointPath, JobID jobId) {
    return new Path(baseCheckpointPath, jobId.toString());
}

19 View Complete Implementation : JobStatusMessage.java
Copyright Apache License 2.0
Author : apache
/**
 * A simple message that holds the state of a job execution.
 */
public clreplaced JobStatusMessage implements java.io.Serializable {

    private final JobID jobId;

    private final String jobName;

    private final JobStatus jobState;

    private final long startTime;

    public JobStatusMessage(JobID jobId, String jobName, JobStatus jobState, long startTime) {
        this.jobId = jobId;
        this.jobName = jobName;
        this.jobState = jobState;
        this.startTime = startTime;
    }

    public JobID getJobId() {
        return jobId;
    }

    public String getJobName() {
        return jobName;
    }

    public JobStatus getJobState() {
        return jobState;
    }

    public long getStartTime() {
        return startTime;
    }
}

19 View Complete Implementation : PermanentBlobCache.java
Copyright Apache License 2.0
Author : apache
/**
 * Registers use of job-related BLOBs.
 *
 * <p>Using any other method to access BLOBs, e.g. {@link #getFile}, is only valid within
 * calls to <tt>registerJob(JobID)</tt> and {@link #releaseJob(JobID)}.
 *
 * @param jobId
 * 		ID of the job this blob belongs to
 *
 * @see #releaseJob(JobID)
 */
public void registerJob(JobID jobId) {
    checkNotNull(jobId);
    synchronized (jobRefCounters) {
        RefCount ref = jobRefCounters.get(jobId);
        if (ref == null) {
            ref = new RefCount();
            jobRefCounters.put(jobId, ref);
        } else {
            // reset cleanup timeout
            ref.keepUntil = -1;
        }
        ++ref.references;
    }
}

19 View Complete Implementation : TransientBlobCache.java
Copyright Apache License 2.0
Author : apache
@Override
protected File getFileInternal(@Nullable JobID jobId, BlobKey blobKey) throws IOException {
    File file = super.getFileInternal(jobId, blobKey);
    readWriteLock.readLock().lock();
    try {
        // regarding concurrent operations, it is not really important which timestamp makes
        // it into the map as they are close to each other anyway, also we can simply
        // overwrite old values as long as we are in the read (or write) lock
        blobExpiryTimes.put(Tuple2.of(jobId, (TransientBlobKey) blobKey), System.currentTimeMillis() + cleanupInterval);
    } finally {
        readWriteLock.readLock().unlock();
    }
    return file;
}

19 View Complete Implementation : Dispatcher.java
Copyright Apache License 2.0
Author : apache
/**
 * Checks whether the given job has already been submitted or executed.
 *
 * @param jobId identifying the submitted job
 * @return true if the job has already been submitted (is running) or has been executed
 * @throws FlinkException if the job scheduling status cannot be retrieved
 */
private boolean isDuplicateJob(JobID jobId) throws FlinkException {
    final RunningJobsRegistry.JobSchedulingStatus jobSchedulingStatus;
    try {
        jobSchedulingStatus = runningJobsRegistry.getJobSchedulingStatus(jobId);
    } catch (IOException e) {
        throw new FlinkException(String.format("Failed to retrieve job scheduling status for job %s.", jobId), e);
    }
    return jobSchedulingStatus == RunningJobsRegistry.JobSchedulingStatus.DONE || jobManagerRunnerFutures.containsKey(jobId);
}

19 View Complete Implementation : TaskManagerSlot.java
Copyright Apache License 2.0
Author : apache
public void completeAllocation(AllocationID allocationId, JobID jobId) {
    Preconditions.checkNotNull(allocationId, "Allocation id must not be null.");
    Preconditions.checkNotNull(jobId, "Job id must not be null.");
    Preconditions.checkState(state == State.PENDING, "In order to complete an allocation, the slot has to be allocated.");
    Preconditions.checkState(Objects.equals(allocationId, replacedignedSlotRequest.getAllocationId()), "Mismatch between allocation id of the pending slot request.");
    state = State.ALLOCATED;
    this.allocationId = allocationId;
    this.jobId = jobId;
    replacedignedSlotRequest = null;
}

19 View Complete Implementation : AbstractExecutionGraphHandler.java
Copyright Apache License 2.0
Author : apache
@Override
protected CompletableFuture<R> handleRequest(@Nonnull HandlerRequest<EmptyRequestBody, M> request, @Nonnull RestfulGateway gateway) throws RestHandlerException {
    JobID jobId = request.getPathParameter(JobIDPathParameter.clreplaced);
    CompletableFuture<AccessExecutionGraph> executionGraphFuture = executionGraphCache.getExecutionGraph(jobId, gateway);
    return executionGraphFuture.thenApplyAsync(executionGraph -> {
        try {
            return handleRequest(request, executionGraph);
        } catch (RestHandlerException rhe) {
            throw new CompletionException(rhe);
        }
    }, executor).exceptionally(throwable -> {
        throwable = ExceptionUtils.stripCompletionException(throwable);
        if (throwable instanceof FlinkJobNotFoundException) {
            throw new CompletionException(new NotFoundException(String.format("Job %s not found", jobId), throwable));
        } else {
            throw new CompletionException(throwable);
        }
    });
}

19 View Complete Implementation : BlobServerGetTest.java
Copyright Apache License 2.0
Author : apache
/**
 * Retrieves the given blob.
 *
 * <p>Note that if a {@link BlobCacheService} is used, it may try to access the {@link
 * BlobServer} to retrieve the blob.
 *
 * @param service
 * 		BLOB client to use for connecting to the BLOB service
 * @param jobId
 * 		job ID or <tt>null</tt> if job-unrelated
 * @param key
 * 		key identifying the BLOB to request
 */
static File get(BlobService service, @Nullable JobID jobId, BlobKey key) throws IOException {
    if (key instanceof PermanentBlobKey) {
        return service.getPermanentBlobService().getFile(jobId, (PermanentBlobKey) key);
    } else if (jobId == null) {
        return service.getTransientBlobService().getFile((TransientBlobKey) key);
    } else {
        return service.getTransientBlobService().getFile(jobId, (TransientBlobKey) key);
    }
}

19 View Complete Implementation : WebMonitorMessagesTest.java
Copyright Apache License 2.0
Author : apache
private Collection<JobDetails> randomJobDetails(Random rnd) {
    final JobDetails[] details = new JobDetails[rnd.nextInt(10)];
    for (int k = 0; k < details.length; k++) {
        int[] numVerticesPerState = new int[ExecutionState.values().length];
        int numTotal = 0;
        for (int i = 0; i < numVerticesPerState.length; i++) {
            int count = rnd.nextInt(55);
            numVerticesPerState[i] = count;
            numTotal += count;
        }
        long time = rnd.nextLong();
        long endTime = rnd.nextBoolean() ? -1L : time + rnd.nextInt();
        long lastModified = endTime == -1 ? time + rnd.nextInt() : endTime;
        String name = new GenericMessageTester.StringInstantiator().instantiate(rnd);
        JobID jid = new JobID();
        JobStatus status = JobStatus.values()[rnd.nextInt(JobStatus.values().length)];
        details[k] = new JobDetails(jid, name, time, endTime, endTime - time, status, lastModified, numVerticesPerState, numTotal);
    }
    return Arrays.asList(details);
}

19 View Complete Implementation : BlobCacheCleanupTest.java
Copyright Apache License 2.0
Author : apache
/**
 * Checks that BLOBs for the given <tt>jobId</tt> are cleaned up eventually (after calling
 * {@link PermanentBlobCache#releaseJob(JobID)}, which is not done by this method!) (waits at
 * most 30s).
 *
 * @param cache
 * 		BLOB server
 * @param jobId
 * 		job ID or <tt>null</tt> if job-unrelated
 * @param keys
 * 		keys identifying BLOBs which were previously registered for the <tt>jobId</tt>
 */
static void verifyJobCleanup(PermanentBlobCache cache, JobID jobId, List<? extends BlobKey> keys) throws InterruptedException, IOException {
    // because we cannot guarantee that there are not thread races in the build system, we
    // loop for a certain while until the references disappear
    {
        long deadline = System.currentTimeMillis() + 30_000L;
        do {
            Thread.sleep(100);
        } while (checkFilesExist(jobId, keys, cache, false) != 0 && System.currentTimeMillis() < deadline);
    }
    // the blob cache should no longer contain the files
    // this fails if we exited via a timeout
    checkFileCountForJob(0, jobId, cache);
}

19 View Complete Implementation : JobManagerMetricGroup.java
Copyright Apache License 2.0
Author : apache
public void removeJob(JobID jobId) {
    if (jobId == null) {
        return;
    }
    synchronized (this) {
        JobManagerJobMetricGroup containedGroup = jobs.remove(jobId);
        if (containedGroup != null) {
            containedGroup.close();
        }
    }
}

19 View Complete Implementation : TaskSlotTable.java
Copyright Apache License 2.0
Author : apache
/**
 * Try to mark the specified slot as active if it has been allocated by the given job.
 *
 * @param jobId of the allocated slot
 * @param allocationId identifying the allocation
 * @return True if the task slot could be marked active.
 */
public boolean tryMarkSlotActive(JobID jobId, AllocationID allocationId) {
    TaskSlot taskSlot = getTaskSlot(allocationId);
    if (taskSlot != null && taskSlot.isAllocated(jobId, allocationId)) {
        return taskSlot.markActive();
    } else {
        return false;
    }
}

19 View Complete Implementation : JobManagerJobGroupTest.java
Copyright Apache License 2.0
Author : apache
@Test
public void testCreateQueryServiceMetricInfo() {
    JobID jid = new JobID();
    MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.defaultMetricRegistryConfiguration());
    JobManagerMetricGroup jm = new JobManagerMetricGroup(registry, "host");
    JobManagerJobMetricGroup jmj = new JobManagerJobMetricGroup(registry, jm, jid, "jobname");
    QueryScopeInfo.JobQueryScopeInfo info = jmj.createQueryServiceMetricInfo(new DummyCharacterFilter());
    replacedertEquals("", info.scope);
    replacedertEquals(jid.toString(), info.jobID);
}

19 View Complete Implementation : StandaloneCompletedCheckpointStoreTest.java
Copyright Apache License 2.0
Author : apache
@Test
public void testPreferCheckpointWithoutSavepoint() throws Exception {
    StandaloneCompletedCheckpointStore store = new StandaloneCompletedCheckpointStore(5);
    JobID jobId = new JobID();
    store.addCheckpoint(checkpoint(jobId, 1L));
    store.addCheckpoint(checkpoint(jobId, 2L));
    store.addCheckpoint(checkpoint(jobId, 3L));
    CompletedCheckpoint latestCheckpoint = store.getLatestCheckpoint(true);
    replacedertThat(latestCheckpoint.getCheckpointID(), equalTo(3L));
}

19 View Complete Implementation : BlobServerGetTest.java
Copyright Apache License 2.0
Author : apache
/**
 * Checks that the given blob does not exist anymore by trying to access it.
 *
 * <p>Note that if a {@link BlobCacheService} is used, it may try to access the {@link
 * BlobServer} to retrieve the blob.
 *
 * @param service
 * 		BLOB client to use for connecting to the BLOB service
 * @param jobId
 * 		job ID or <tt>null</tt> if job-unrelated
 * @param key
 * 		key identifying the BLOB to request
 */
static void verifyDeleted(BlobService service, @Nullable JobID jobId, BlobKey key) throws IOException {
    try {
        get(service, jobId, key);
        fail("File " + jobId + "/" + key + " should have been deleted.");
    } catch (IOException e) {
    // expected
    }
}

19 View Complete Implementation : ProgramTargetDescriptor.java
Copyright Apache License 2.0
Author : apache
/**
 * Describes the target where a table program has been submitted to.
 */
public clreplaced ProgramTargetDescriptor {

    private final JobID jobId;

    public ProgramTargetDescriptor(JobID jobId) {
        this.jobId = checkNotNull(jobId);
    }

    public JobID getJobId() {
        return jobId;
    }

    @Override
    public String toString() {
        return String.format("Job ID: %s\n", jobId);
    }

    /**
     * Creates a program target description from deployment clreplacedes.
     *
     * @param jobId job id
     * @return program target descriptor
     */
    public static ProgramTargetDescriptor of(JobID jobId) {
        return new ProgramTargetDescriptor(jobId);
    }
}

19 View Complete Implementation : KvStateClientProxyImplTest.java
Copyright Apache License 2.0
Author : apache
/**
 * Tests that {@link KvStateLocationOracle} registered under {@link HighAvailabilityServices#DEFAULT_JOB_ID}
 * will be used for all requests.
 */
@Test
public void testLegacyCodePathPreference() {
    final TestingKvStateLocationOracle kvStateLocationOracle = new TestingKvStateLocationOracle();
    kvStateClientProxy.updateKvStateLocationOracle(HighAvailabilityServices.DEFAULT_JOB_ID, kvStateLocationOracle);
    final JobID jobId = new JobID();
    kvStateClientProxy.updateKvStateLocationOracle(jobId, new TestingKvStateLocationOracle());
    replacedertThat(kvStateClientProxy.getKvStateLocationOracle(jobId), equalTo(kvStateLocationOracle));
}

19 View Complete Implementation : BlobServerDeleteTest.java
Copyright Apache License 2.0
Author : apache
/**
 * Deletes a transient BLOB from the given BLOB service.
 *
 * @param service
 * 		blob service
 * @param jobId
 * 		job ID (or <tt>null</tt> if job-unrelated)
 * @param key
 * 		blob key
 *
 * @return delete success
 */
static boolean delete(BlobService service, @Nullable JobID jobId, TransientBlobKey key) {
    if (jobId == null) {
        return service.getTransientBlobService().deleteFromCache(key);
    } else {
        return service.getTransientBlobService().deleteFromCache(jobId, key);
    }
}