org.apache.spark.Accumulator.value()方法的使用及代码示例

x33g5p2x  于2022-01-16 转载在 其他  
字(6.1k)|赞(0)|评价(0)|浏览(117)

本文整理了Java中org.apache.spark.Accumulator.value()方法的一些代码示例,展示了Accumulator.value()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Accumulator.value()方法的具体详情如下:
包路径:org.apache.spark.Accumulator
类名称:Accumulator
方法名:value

Accumulator.value介绍

暂无

代码示例

代码示例来源:origin: apache/hive

public long getValue() {
 if (accumulator != null) {
  return accumulator.value();
 } else {
  return accumValue;
 }
}

代码示例来源:origin: apache/hive

SparkCounter snapshot() {
 return new SparkCounter(name, displayName, accumulator.value());
}

代码示例来源:origin: databricks/learning-spark

System.out.println("Lines with 'KK6JKQ': " + count.value());
  }});
callSigns.saveAsTextFile(outputDir + "/callsigns");
System.out.println("Blank lines: "+ blankLines.value());
if (invalidSignCount.value() < 0.1 * validSignCount.value()) {
 contactCounts.saveAsTextFile(outputDir + "/contactCount");
} else {
 System.out.println("Too many errors " + invalidSignCount.value() +
           " for " + validSignCount.value());
 System.exit(1);

代码示例来源:origin: org.apache.spark/spark-core

assertEquals((Integer) 25, intAccum.value());
assertEquals((Double) 25.0, doubleAccum.value());
assertEquals((Float) 25.0f, floatAccum.value());
assertEquals((Float) 5.0f, floatAccum.value());

代码示例来源:origin: org.apache.spark/spark-core_2.10

assertEquals((Integer) 25, intAccum.value());
assertEquals((Double) 25.0, doubleAccum.value());
assertEquals((Float) 25.0f, floatAccum.value());
assertEquals((Float) 5.0f, floatAccum.value());

代码示例来源:origin: org.apache.spark/spark-core_2.11

assertEquals((Integer) 25, intAccum.value());
assertEquals((Double) 25.0, doubleAccum.value());
assertEquals((Float) 25.0f, floatAccum.value());
assertEquals((Float) 5.0f, floatAccum.value());

代码示例来源:origin: apache/tinkerpop

@Override
public Set<String> keys() {
  if (this.inExecute)
    return this.broadcast.getValue().keySet();
  else {
    final Set<String> trueKeys = new HashSet<>();
    this.sparkMemory.forEach((key, value) -> {
      if (!value.value().isEmpty())
        trueKeys.add(key);
    });
    return Collections.unmodifiableSet(trueKeys);
  }
}

代码示例来源:origin: apache/tinkerpop

protected void broadcastMemory(final JavaSparkContext sparkContext) {
  this.broadcast.destroy(true); // do we need to block?
  final Map<String, Object> toBroadcast = new HashMap<>();
  this.sparkMemory.forEach((key, object) -> {
    if (!object.value().isEmpty() && this.memoryComputeKeys.get(key).isBroadcast())
      toBroadcast.put(key, object.value());
  });
  this.broadcast = sparkContext.broadcast(toBroadcast);
}

代码示例来源:origin: apache/tinkerpop

@Override
public <R> R get(final String key) throws IllegalArgumentException {
  if (!this.memoryComputeKeys.containsKey(key))
    throw Memory.Exceptions.memoryDoesNotExist(key);
  if (this.inExecute && !this.memoryComputeKeys.get(key).isBroadcast())
    throw Memory.Exceptions.memoryDoesNotExist(key);
  final ObjectWritable<R> r = (ObjectWritable<R>) (this.inExecute ? this.broadcast.value().get(key) : this.sparkMemory.get(key).value());
  if (null == r || r.isEmpty())
    throw Memory.Exceptions.memoryDoesNotExist(key);
  else
    return r.get();
}

代码示例来源:origin: org.apache.pig/pig

public T getValue() {
  if (accumulator != null) {
    return accumulator.value();
  } else {
    return null;
  }
}

代码示例来源:origin: com.github.hyukjinkwon/spark-client

public long getValue() {
 if (accumulator != null) {
  return accumulator.value();
 } else {
  return accumValue;
 }
}

代码示例来源:origin: org.apache.beam/beam-runners-spark

private static void checkpoint() throws IOException {
 if (checkpointFilePath != null) {
  Checkpoint.writeObject(fileSystem, checkpointFilePath, instance.value());
 }
}

代码示例来源:origin: org.apache.beam/beam-runners-spark

private static void checkpoint() throws IOException {
 if (checkpointFilePath != null) {
  Checkpoint.writeObject(fileSystem, checkpointFilePath, instance.value());
 }
}

代码示例来源:origin: ai.grakn/grakn-kb

protected void broadcastMemory(final JavaSparkContext sparkContext) {
  this.broadcast.destroy(true); // do we need to block?
  final Map<String, Object> toBroadcast = new HashMap<>();
  this.sparkMemory.forEach((key, object) -> {
    if (!object.value().isEmpty() && this.memoryComputeKeys.get(key).isBroadcast()) {
      toBroadcast.put(key, object.value());
    }
  });
  this.broadcast = sparkContext.broadcast(toBroadcast);
}

代码示例来源:origin: org.apache.tinkerpop/spark-gremlin

protected void broadcastMemory(final JavaSparkContext sparkContext) {
  this.broadcast.destroy(true); // do we need to block?
  final Map<String, Object> toBroadcast = new HashMap<>();
  this.sparkMemory.forEach((key, object) -> {
    if (!object.value().isEmpty() && this.memoryComputeKeys.get(key).isBroadcast())
      toBroadcast.put(key, object.value());
  });
  this.broadcast = sparkContext.broadcast(toBroadcast);
}

代码示例来源:origin: apache/crunch

private Counters getCounters() {
 Counters c = new Counters();
 Map<String, Map<String, Long>> values = counters.value();
 for (Map.Entry<String, Map<String, Long>> e : values.entrySet()) {
  CounterGroup cg = c.getGroup(e.getKey());
  for (Map.Entry<String, Long> f : e.getValue().entrySet()) {
   cg.findCounter(f.getKey()).setValue(f.getValue());
  }
 }
 return c;
}

代码示例来源:origin: org.qcri.rheem/rheem-spark

@Override
public OptionalLong getMeasuredCardinality() {
  if (this.accumulator != null) {
    this.setMeasuredCardinality(this.accumulator.value());
  }
  return super.getMeasuredCardinality();
}

代码示例来源:origin: org.apache.beam/beam-runners-spark

@Override
public MetricResults metrics() {
 return asAttemptedOnlyMetricResults(MetricsAccumulator.getInstance().value());
}

代码示例来源:origin: org.qcri.rheem/rheem-spark

@Override
protected void doDispose() {
  if (this.accumulator != null) {
    this.setMeasuredCardinality(this.accumulator.value());
    this.accumulator = null;
  }
  if (this.isRddCached() && this.rdd != null) {
    Actions.doSafe(this.rdd::unpersist);
    logger.debug("Unpersisted {}.", this.rdd);
    this.rdd = null;
  }
}

代码示例来源:origin: org.apache.tinkerpop/spark-gremlin

@Override
public <R> R get(final String key) throws IllegalArgumentException {
  if (!this.memoryComputeKeys.containsKey(key))
    throw Memory.Exceptions.memoryDoesNotExist(key);
  if (this.inExecute && !this.memoryComputeKeys.get(key).isBroadcast())
    throw Memory.Exceptions.memoryDoesNotExist(key);
  final ObjectWritable<R> r = (ObjectWritable<R>) (this.inExecute ? this.broadcast.value().get(key) : this.sparkMemory.get(key).value());
  if (null == r || r.isEmpty())
    throw Memory.Exceptions.memoryDoesNotExist(key);
  else
    return r.get();
}

相关文章