org.apache.helix.task.WorkflowContext类的使用及代码示例

x33g5p2x  于2022-02-03 转载在 其他  
字(12.8k)|赞(0)|评价(0)|浏览(147)

本文整理了Java中org.apache.helix.task.WorkflowContext类的一些代码示例，展示了WorkflowContext类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。WorkflowContext类的具体详情如下：
包路径：org.apache.helix.task.WorkflowContext
类名称：WorkflowContext

WorkflowContext介绍

[英]Typed interface to the workflow context information stored by TaskRebalancer in the Helix property store
[中]TaskRebalance在Helix属性存储中存储的工作流上下文信息的类型化接口

代码示例

代码示例来源：origin: apache/incubator-gobblin

WorkflowContext workflowContext = TaskDriver.getWorkflowContext(helixManager, workFlowName);
if (workflowContext != null) {
 TaskState jobState = workflowContext.getJobState(TaskUtil.getNamespacedJobName(workFlowName, jobName));
 switch (jobState) {
  case STOPPED:

代码示例来源：origin: apache/incubator-pinot

/**
 * Get the task queue state for the given task type.
 *
 * @param taskType Task type
 * @return Task queue state
 */
public synchronized TaskState getTaskQueueState(@Nonnull String taskType) {
 return _taskDriver.getWorkflowContext(getHelixJobQueueName(taskType)).getWorkflowState();
}

代码示例来源：origin: apache/incubator-pinot

/**
 * Get all tasks for the given task type.
 *
 * @param taskType Task type
 * @return Set of task names
 */
@Nonnull
public synchronized Set<String> getTasks(@Nonnull String taskType) {
 Set<String> helixJobs = _taskDriver.getWorkflowContext(getHelixJobQueueName(taskType)).getJobStates().keySet();
 Set<String> tasks = new HashSet<>(helixJobs.size());
 for (String helixJobName : helixJobs) {
  tasks.add(getPinotTaskName(helixJobName));
 }
 return tasks;
}

代码示例来源：origin: org.apache.helix/helix-core

boolean incomplete = false;
TaskState workflowState = ctx.getWorkflowState();
if (TaskState.TIMED_OUT.equals(workflowState)) {
 TaskState jobState = ctx.getJobState(job);
 if (jobState == TaskState.FAILED || jobState == TaskState.TIMED_OUT) {
  failedJobs++;
  if (!cfg.isJobQueue() && failedJobs > cfg.getFailureThreshold()) {
   ctx.setWorkflowState(TaskState.FAILED);
   LOG.info("Workflow {} reached the failure threshold, so setting its state to FAILED.", cfg.getWorkflowId());
   for (String jobToFail : cfg.getJobDag().getAllNodes()) {
    if (ctx.getJobState(jobToFail) == TaskState.IN_PROGRESS) {
     ctx.setJobState(jobToFail, TaskState.ABORTED);
 ctx.setWorkflowState(TaskState.COMPLETED);
 return true;

代码示例来源：origin: apache/helix

if (!workflowCfg.isJobQueue() && !finalStates.contains(workflowCtx.getWorkflowState())) {
 scheduleRebalanceForTimeout(workflow, workflowCtx.getStartTime(), workflowCfg.getTimeout());
 if (!TaskState.TIMED_OUT.equals(workflowCtx.getWorkflowState()) && isTimeout(workflowCtx.getStartTime(), workflowCfg.getTimeout())) {
  workflowCtx.setWorkflowState(TaskState.TIMED_OUT);
  _taskDataCache.updateWorkflowContext(workflow, workflowCtx);
if (!finalStates.contains(workflowCtx.getWorkflowState()) && TargetState.STOP.equals(targetState)) {
 LOG.info("Workflow " + workflow + "is marked as stopped.");
 if (isWorkflowStopped(workflowCtx, workflowCfg)) {
  workflowCtx.setWorkflowState(TaskState.STOPPED);
  _taskDataCache.updateWorkflowContext(workflow, workflowCtx);
if (workflowCtx.getFinishTime() == WorkflowContext.UNFINISHED && isWorkflowFinished(workflowCtx,
  workflowCfg, _taskDataCache.getJobConfigMap(), _taskDataCache)) {
 workflowCtx.setFinishTime(currentTime);
 updateWorkflowMonitor(workflowCtx, workflowCfg);
 _taskDataCache.updateWorkflowContext(workflow, workflowCtx);
if (workflowCtx.getFinishTime() != WorkflowContext.UNFINISHED) {
 LOG.info("Workflow " + workflow + " is finished.");
 long expiryTime = workflowCfg.getExpiry();
 if (workflowCtx.getFinishTime() + expiryTime <= currentTime) {
  LOG.info("Workflow " + workflow + " passed expiry time, cleaning up the workflow context.");
  cleanupWorkflow(workflow);
 } else {

代码示例来源：origin: apache/helix

public static WorkflowContext buildWorkflowContext(String workflowResource,
  TaskState workflowState, Long startTime, TaskState... jobStates) {
 WorkflowContext workflowContext =
   new WorkflowContext(new ZNRecord(TaskUtil.WORKFLOW_CONTEXT_KW));
 workflowContext.setName(workflowResource);
 workflowContext.setStartTime(startTime == null ? System.currentTimeMillis() : startTime);
 int jobId = 0;
 for (TaskState jobstate : jobStates) {
  workflowContext
    .setJobState(TaskUtil.getNamespacedJobName(workflowResource, JOB_KW) + jobId++, jobstate);
 }
 workflowContext.setWorkflowState(workflowState);
 return workflowContext;
}

代码示例来源：origin: org.apache.helix/helix-core

if (!workflowCfg.isJobQueue() && !finalStates.contains(workflowCtx.getWorkflowState())) {
 scheduleRebalanceForTimeout(workflow, workflowCtx.getStartTime(), workflowCfg.getTimeout());
 if (!TaskState.TIMED_OUT.equals(workflowCtx.getWorkflowState())
   && isTimeout(workflowCtx.getStartTime(), workflowCfg.getTimeout())) {
  workflowCtx.setWorkflowState(TaskState.TIMED_OUT);
  clusterData.updateWorkflowContext(workflow, workflowCtx, _manager.getHelixDataAccessor());
if (!finalStates.contains(workflowCtx.getWorkflowState())
  && TargetState.STOP.equals(targetState)) {
 LOG.info("Workflow " + workflow + "is marked as stopped.");
 if (isWorkflowStopped(workflowCtx, workflowCfg)) {
  workflowCtx.setWorkflowState(TaskState.STOPPED);
  clusterData.updateWorkflowContext(workflow, workflowCtx, _manager.getHelixDataAccessor());
if (workflowCtx.getFinishTime() == WorkflowContext.UNFINISHED && isWorkflowFinished(workflowCtx,
  workflowCfg, clusterData.getJobConfigMap(), clusterData)) {
 workflowCtx.setFinishTime(currentTime);
 updateWorkflowMonitor(workflowCtx, workflowCfg);
 clusterData.updateWorkflowContext(workflow, workflowCtx, _manager.getHelixDataAccessor());
if (workflowCtx.getFinishTime() != WorkflowContext.UNFINISHED) {
 LOG.info("Workflow " + workflow + " is finished.");
 long expiryTime = workflowCfg.getExpiry();
 if (workflowCtx.getFinishTime() + expiryTime <= currentTime) {
  LOG.info("Workflow " + workflow + " passed expiry time, cleaning up the workflow context.");
  cleanupWorkflow(workflow, workflowCfg);

代码示例来源：origin: org.apache.helix/helix-core

long timeToSchedule = Long.MAX_VALUE;
for (String job : workflowCfg.getJobDag().getAllNodes()) {
 TaskState jobState = workflowCtx.getJobState(job);
 if (jobState != null && !jobState.equals(TaskState.NOT_STARTED)) {
  if (LOG.isDebugEnabled()) {
  long calculatedStartTime = workflowCtx.getJobStartTime(job);
  if (calculatedStartTime < 0) {
   workflowCtx.setJobStartTime(job, calculatedStartTime);
  } else {
   scheduleSingleJob(job, jobConfig);
   workflowCtx.setJobState(job, TaskState.NOT_STARTED);
   scheduledJobs++;

代码示例来源：origin: apache/helix

/**
 * Remove all jobs that are in final states (ABORTED, FAILED, COMPLETED) from the job queue. The
 * job config, job context will be removed from Zookeeper.
 *
 * @param queue The name of job queue
 */
public void cleanupQueue(String queue) {
 WorkflowConfig workflowConfig = TaskUtil.getWorkflowConfig(_accessor, queue);
 if (workflowConfig == null) {
  throw new IllegalArgumentException("Queue " + queue + " does not yet exist!");
 }
 boolean isTerminable = workflowConfig.isTerminable();
 if (isTerminable) {
  throw new IllegalArgumentException(queue + " is not a queue!");
 }
 WorkflowContext wCtx = TaskUtil.getWorkflowContext(_propertyStore, queue);
 if (wCtx == null || wCtx.getWorkflowState() == null) {
  throw new IllegalStateException("Queue " + queue + " does not have a valid work state!");
 }
 Set<String> jobs = new HashSet<String>();
 for (String jobNode : workflowConfig.getJobDag().getAllNodes()) {
  TaskState curState = wCtx.getJobState(jobNode);
  if (curState != null && (curState == TaskState.ABORTED || curState == TaskState.COMPLETED
    || curState == TaskState.FAILED)) {
   jobs.add(jobNode);
  }
 }
 TaskUtil.removeJobsFromWorkflow(_accessor, _propertyStore, queue, jobs, true);
}

代码示例来源：origin: apache/helix

Assert.assertNull(context.getJobState(notStartedJobName));
Assert.assertTrue(context.getFinishTime() - context.getStartTime() >= timeout);

代码示例来源：origin: apache/helix

Thread.sleep(timeToSleep);
  ctx = getWorkflowContext(workflowName);
 } while ((ctx == null || ctx.getLastScheduledSingleWorkflow() == null));
 workflowName = ctx.getLastScheduledSingleWorkflow();
 Thread.sleep(timeToSleep);
 ctx = getWorkflowContext(workflowName);
} while ((ctx == null || ctx.getJobState(jobName) == null || !allowedStates
  .contains(ctx.getJobState(jobName))) && System.currentTimeMillis() < st + timeout);
if (ctx == null || !allowedStates.contains(ctx.getJobState(jobName))) {
 throw new HelixException(
   String.format("Workflow \"%s\" context is null or job \"%s\" is not in states: %s",
return ctx.getJobState(jobName);

代码示例来源：origin: apache/helix

@Override
 public ZNRecord update(ZNRecord currentData) {
  if (currentData != null) {
   WorkflowContext workflowContext = new WorkflowContext(currentData);
   workflowContext.removeJobStates(jobs);
   workflowContext.removeJobStartTime(jobs);
   currentData = workflowContext.getRecord();
  }
  return currentData;
 }
};

代码示例来源：origin: org.apache.helix/helix-core

protected void updateWorkflowMonitor(WorkflowContext context, WorkflowConfig config) {
 if (_clusterStatusMonitor != null) {
  _clusterStatusMonitor.updateWorkflowCounters(config, context.getWorkflowState(),
    context.getFinishTime() - context.getStartTime());
 }
}

代码示例来源：origin: org.apache.helix/helix-core

workflowCtx.setJobState(jobResource, TaskState.STOPPED);
 workflowCtx.setWorkflowState(TaskState.STOPPED);
workflowCtx.setJobState(jobResource, TaskState.IN_PROGRESS);
workflowCtx.setWorkflowState(TaskState.IN_PROGRESS);
    workflowCtx.setJobState(jobResource, TaskState.FAILED);
    if (workflowConfig.isTerminable()) {
     workflowCtx.setWorkflowState(TaskState.FAILED);
     workflowCtx.setFinishTime(finishTime);
workflowCtx.setJobState(jobResource, TaskState.COMPLETED);
jobCtx.setFinishTime(currentTime);
if (isWorkflowComplete(workflowCtx, workflowConfig)) {
 workflowCtx.setWorkflowState(TaskState.COMPLETED);
 workflowCtx.setFinishTime(currentTime);

代码示例来源：origin: com.linkedin.gobblin/gobblin-cluster

private void markJobComplete(String jobName, JobContext jobContext,
  WorkflowConfig workflowConfig, WorkflowContext workflowContext) {
 long currentTime = System.currentTimeMillis();
 workflowContext.setJobState(jobName, TaskState.COMPLETED);
 jobContext.setFinishTime(currentTime);
 if (isWorkflowFinished(workflowContext, workflowConfig)) {
  workflowContext.setFinishTime(currentTime);
 }
}

代码示例来源：origin: apache/helix

/**
 * Update context of the Workflow
 */
public void updateWorkflowContext(String resourceName, WorkflowContext workflowContext) {
 updateContext(resourceName, workflowContext.getRecord());
}

代码示例来源：origin: org.apache.helix/helix-core

/**
 * Helper function to change target state for a given workflow
 */
private void setWorkflowTargetState(String workflow, TargetState state) {
 setSingleWorkflowTargetState(workflow, state);
 // For recurring schedules, last scheduled incomplete workflow must also be handled
 WorkflowContext wCtx = TaskUtil.getWorkflowContext(_propertyStore, workflow);
 if (wCtx != null) {
  String lastScheduledWorkflow = wCtx.getLastScheduledSingleWorkflow();
  if (lastScheduledWorkflow != null) {
   setSingleWorkflowTargetState(lastScheduledWorkflow, state);
  }
 }
}

代码示例来源：origin: apache/helix

@Test public void testTaskRetryWithoutDelay() throws Exception {
  String jobResource = TestHelper.getTestMethodName();
  JobConfig.Builder jobBuilder = JobConfig.Builder.fromMap(WorkflowGenerator.DEFAULT_JOB_CONFIG);
  jobBuilder.setJobCommandConfigMap(WorkflowGenerator.DEFAULT_COMMAND_CONFIG)
    .setMaxAttemptsPerTask(2).setCommand(MockTask.TASK_COMMAND)
    .setFailureThreshold(Integer.MAX_VALUE)
    .setJobCommandConfigMap(ImmutableMap.of(MockTask.FAILURE_COUNT_BEFORE_SUCCESS, "1"));
  Workflow flow =
    WorkflowGenerator.generateSingleJobWorkflowBuilder(jobResource, jobBuilder).build();
  _driver.start(flow);

  // Wait until the job completes.
  _driver.pollForWorkflowState(jobResource, TaskState.COMPLETED);

  long startTime = _driver.getWorkflowContext(jobResource).getStartTime();
  long finishedTime = _driver.getWorkflowContext(jobResource).getFinishTime();

  // It should finished at less than 2 sec
  Assert.assertTrue(finishedTime - startTime <= 2000L);
 }
}

代码示例来源：origin: apache/helix

public WorkflowContext getOrInitializeWorkflowContext(
  String workflowName, TaskDataCache cache) {
 WorkflowContext workflowCtx = cache.getWorkflowContext(workflowName);
 if (workflowCtx == null) {
  workflowCtx = new WorkflowContext(new ZNRecord(TaskUtil.WORKFLOW_CONTEXT_KW));
  workflowCtx.setStartTime(System.currentTimeMillis());
  workflowCtx.setName(workflowName);
  LOG.debug("Workflow context is created for " + workflowName);
 }
 return workflowCtx;
}

代码示例来源：origin: apache/helix

/**
 * This test method tests whether PropertyKey.Builder successfully creates a path for
 * WorkflowContext instances.
 * TODO: KeyBuilder must handle the case for future versions of Task Framework with a different
 * path structure
 */
@Test
public void testGetWorkflowContext() {
 // Manually create a WorkflowContext instance
 ZNRecord znRecord = new ZNRecord(WORKFLOW_NAME);
 WorkflowContext workflowContext = new WorkflowContext(znRecord);
 _manager.getHelixPropertyStore().set(
   Joiner.on("/").join(TaskConstants.REBALANCER_CONTEXT_ROOT, WORKFLOW_NAME, CONTEXT_NODE),
   workflowContext.getRecord(), AccessOption.PERSISTENT);
 // Test retrieving this WorkflowContext using PropertyKey.Builder.getPath()
 String path = KEY_BUILDER.workflowContext(WORKFLOW_NAME).getPath();
 WorkflowContext workflowCtx =
   new WorkflowContext(_baseAccessor.get(path, null, AccessOption.PERSISTENT));
 Assert.assertEquals(workflowContext, workflowCtx);
}

内容来源于网络，如有侵权，请联系作者删除！

WorkflowContext

相关文章

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com