org.apache.hadoop.util.Progress类的使用及代码示例

x33g5p2x  于2022-01-26 转载在 其他  
字(6.8k)|赞(0)|评价(0)|浏览(118)

本文整理了Java中org.apache.hadoop.util.Progress类的一些代码示例,展示了Progress类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Progress类的具体详情如下:
包路径:org.apache.hadoop.util.Progress
类名称:Progress

Progress介绍

[英]Utility to assist with generation of progress reports. Applications build a hierarchy of Progress instances, each modelling a phase of execution. The root is constructed with #Progress(). Nodes for sub-phases are created by calling #addPhase().
[中]用于帮助生成进度报告的实用程序。应用程序构建了一个进度实例的层次结构,每个实例对一个执行阶段进行建模。根由#Progress()构成。子阶段的节点是通过调用#addPhase()创建的。

代码示例

代码示例来源:origin: org.apache.hadoop/hadoop-common

private void updateProgress(long bytesProcessed) {
 totalBytesProcessed += bytesProcessed;
 if (progPerByte > 0) {
  mergeProgress.set(totalBytesProcessed * progPerByte);
 }
}

代码示例来源:origin: org.apache.hadoop/hadoop-common

/** Adds a new phase. Caller needs to set progress weightage */
private synchronized Progress addNewPhase() {
 Progress phase = new Progress();
 phases.add(phase);
 phase.setParent(this);
 return phase;
}

代码示例来源:origin: org.apache.hadoop/hadoop-common

private synchronized void toString(StringBuilder buffer) {
 buffer.append(status);
 if (phases.size() != 0 && currentPhase < phases.size()) {
  buffer.append(" > ");
  phase().toString(buffer);
 }
}

代码示例来源:origin: org.apache.hadoop/hadoop-common

/** Adds a named node to the tree. */
public Progress addPhase(String status) {
 Progress phase = addPhase();
 phase.setStatus(status);
 return phase;
}

代码示例来源:origin: ch.cern.hadoop/hadoop-mapreduce-client-core

mapOutputFile.getOutputIndexFileForWriteInVolume(filename[0]), job);
sortPhase.complete();
return;
 finalOut.close();
sortPhase.complete();
return;
sortPhase.addPhases(partitions); // Divide sort phase into sub-phases
         new Path(mapId.toString()),
         job.getOutputKeyComparator(), reporter, sortSegments,
         null, spilledRecordsCounter, sortPhase.phase(),
         TaskType.MAP);
 sortPhase.startNextPhase();

代码示例来源:origin: ch.cern.hadoop/hadoop-mapreduce-client-core

public void informReduceProgress() {
  reducePhase.set(super.in.getProgress().getProgress()); // update progress
  reporter.progress();
 }
}

代码示例来源:origin: com.facebook.hadoop/hadoop-core

private void sendLastUpdate(TaskUmbilicalProtocol umbilical) 
throws IOException {
 // send a final status report
 taskStatus.statusUpdate(taskProgress.get(),
             taskProgress.toString(), 
             counters);
 statusUpdate(umbilical);
}

代码示例来源:origin: com.github.jiayuhan-it/hadoop-mapreduce-client-core

public void setProgress(float progress) {
 // set current phase progress.
 // This method assumes that task has phases.
 taskProgress.phase().set(progress);
 // indicate that progress update needs to be sent
 setProgressFlag();
}

代码示例来源:origin: org.jvnet.hudson.hadoop/hadoop-core

public void informReduceProgress() {
  reducePhase.set(super.in.getProgress().get()); // update progress
  reporter.progress();
 }
}

代码示例来源:origin: org.apache.hadoop/hadoop-common

/** Computes progress in this node. */
private synchronized float getInternal() {
 int phaseCount = phases.size();
 if (phaseCount != 0) {
  float subProgress = 0.0f;
  float progressFromCurrentPhase = 0.0f;
  if (currentPhase < phaseCount) {
   subProgress = phase().getInternal();
   progressFromCurrentPhase =
    getProgressWeightage(currentPhase) * subProgress;
  }
  
  float progressFromCompletedPhases = 0.0f;
  if (fixedWeightageForAllPhases) { // same progress weightage for each phase
   progressFromCompletedPhases = progressPerPhase * currentPhase;
  }
  else {
   for (int i = 0; i < currentPhase; i++) {
    // progress weightages of phases could be different. Add them
    progressFromCompletedPhases += getProgressWeightage(i);
   }
  }
  return  progressFromCompletedPhases + progressFromCurrentPhase;
 } else {
  return progress;
 }
}

代码示例来源:origin: com.github.jiayuhan-it/hadoop-mapreduce-client-core

public boolean next() throws IOException {
  boolean ret = rawIter.next();
  reporter.setProgress(rawIter.getProgress().getProgress());
  return ret;
 }
};

代码示例来源:origin: org.apache.hadoop/hadoop-common

/** Completes this node, moving the parent node to its next child. */
public void complete() {
 // we have to traverse up to our parent, so be careful about locking.
 Progress myParent;
 synchronized(this) {
  progress = 1.0f;
  myParent = parent;
 }
 if (myParent != null) {
  // this will synchronize on the parent, so we make sure we release
  // our lock before getting the parent's, since we're traversing 
  // against the normal traversal direction used by get() or toString().
  // We don't need transactional semantics, so we're OK doing this. 
  myParent.startNextPhase();
 }
}

代码示例来源:origin: com.github.jiayuhan-it/hadoop-mapreduce-client-core

public void setStatus(String status) {
 taskProgress.setStatus(normalizeStatus(status, conf));
 // indicate that progress update needs to be sent
 setProgressFlag();
}
public void setProgress(float progress) {

代码示例来源:origin: org.apache.hadoop/hadoop-mapred

sortPhase.addPhases(partitions); // Divide sort phase into sub-phases
Merger.considerFinalMergeForProgress();
         new Path(mapId.toString()),
         job.getOutputKeyComparator(), reporter, sortSegments,
         null, spilledRecordsCounter, sortPhase.phase());
 sortPhase.startNextPhase();

代码示例来源:origin: ch.cern.hadoop/hadoop-mapreduce-client-core

copyPhase = getProgress().addPhase("copy");
 sortPhase  = getProgress().addPhase("sort");
 reducePhase = getProgress().addPhase("reduce");
sortPhase.complete();                         // sort is complete
setPhase(TaskStatus.Phase.REDUCE); 
statusUpdate(umbilical);

代码示例来源:origin: org.apache.hadoop/hadoop-mapreduce-client-core

doReturn(mof).when(mockTask).getMapOutputFile();
doReturn(attemptId).when(mockTask).getTaskID();
doReturn(new Progress()).when(mockTask).getSortPhase();
TaskReporter mockReporter = mock(TaskReporter.class);
doReturn(new Counter()).when(mockReporter).getCounter(

代码示例来源:origin: org.apache.tez/tez-runtime-library

@Override
public Progress getProgress() {
 progress.complete();
 return progress;
}

代码示例来源:origin: org.apache.hadoop/hadoop-common

@Override
public String toString() {
 StringBuilder result = new StringBuilder();
 toString(result);
 return result.toString();
}

代码示例来源:origin: org.apache.hadoop/hadoop-mapreduce-client-core

Progress progress = new Progress();
scheduler.tipFailed(taskId1);
Assert.assertEquals("Progress should be 0.5", 0.5f, progress.getProgress(),
  0.0f);
Assert.assertFalse(scheduler.waitUntilDone(1));
Assert.assertEquals("Progress should be 1.0", 1.0f, progress.getProgress(),
  0.0f);
Assert.assertTrue(scheduler.waitUntilDone(1));

代码示例来源:origin: org.apache.hadoop/hadoop-mapreduce-client-core

Progress progress = new Progress();
ShuffleSchedulerImpl<K, V> scheduler = new ShuffleSchedulerImpl<K, V>(job, status, null,
  null, progress, context.getShuffledMapsCounter(),
Assert.assertEquals(copyMessage(1, 1, 1), progress.toString());
Assert.assertEquals(copyMessage(2, 1, 1), progress.toString());
Assert.assertEquals(copyMessage(3, 2, 2), progress.toString());
Assert.assertEquals(copyMessage(4, 0.5, 1), progress.toString());
Assert.assertEquals(copyMessage(5, 1, 1), progress.toString());
Assert.assertEquals(copyMessage(6, 1, 1), progress.toString());
Assert.assertEquals(copyMessage(7, 1, 1), progress.toString());
Assert.assertEquals(copyMessage(8, 0.5, 1), progress.toString());
Assert.assertEquals(copyMessage(9, 1, 1), progress.toString());
Assert.assertEquals(copyMessage(10, 1, 2), progress.toString());

相关文章