本文整理了Java中org.apache.spark.Accumulator.value()
方法的一些代码示例,展示了Accumulator.value()
的具体用法。这些代码示例主要来源于Github
/Stackoverflow
/Maven
等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Accumulator.value()
方法的具体详情如下:
包路径:org.apache.spark.Accumulator
类名称:Accumulator
方法名:value
暂无
代码示例来源:origin: apache/hive
public long getValue() {
if (accumulator != null) {
return accumulator.value();
} else {
return accumValue;
}
}
代码示例来源:origin: apache/hive
SparkCounter snapshot() {
return new SparkCounter(name, displayName, accumulator.value());
}
代码示例来源:origin: databricks/learning-spark
System.out.println("Lines with 'KK6JKQ': " + count.value());
}});
callSigns.saveAsTextFile(outputDir + "/callsigns");
System.out.println("Blank lines: "+ blankLines.value());
if (invalidSignCount.value() < 0.1 * validSignCount.value()) {
contactCounts.saveAsTextFile(outputDir + "/contactCount");
} else {
System.out.println("Too many errors " + invalidSignCount.value() +
" for " + validSignCount.value());
System.exit(1);
代码示例来源:origin: org.apache.spark/spark-core
assertEquals((Integer) 25, intAccum.value());
assertEquals((Double) 25.0, doubleAccum.value());
assertEquals((Float) 25.0f, floatAccum.value());
assertEquals((Float) 5.0f, floatAccum.value());
代码示例来源:origin: org.apache.spark/spark-core_2.10
assertEquals((Integer) 25, intAccum.value());
assertEquals((Double) 25.0, doubleAccum.value());
assertEquals((Float) 25.0f, floatAccum.value());
assertEquals((Float) 5.0f, floatAccum.value());
代码示例来源:origin: org.apache.spark/spark-core_2.11
assertEquals((Integer) 25, intAccum.value());
assertEquals((Double) 25.0, doubleAccum.value());
assertEquals((Float) 25.0f, floatAccum.value());
assertEquals((Float) 5.0f, floatAccum.value());
代码示例来源:origin: apache/tinkerpop
@Override
public Set<String> keys() {
if (this.inExecute)
return this.broadcast.getValue().keySet();
else {
final Set<String> trueKeys = new HashSet<>();
this.sparkMemory.forEach((key, value) -> {
if (!value.value().isEmpty())
trueKeys.add(key);
});
return Collections.unmodifiableSet(trueKeys);
}
}
代码示例来源:origin: apache/tinkerpop
protected void broadcastMemory(final JavaSparkContext sparkContext) {
this.broadcast.destroy(true); // do we need to block?
final Map<String, Object> toBroadcast = new HashMap<>();
this.sparkMemory.forEach((key, object) -> {
if (!object.value().isEmpty() && this.memoryComputeKeys.get(key).isBroadcast())
toBroadcast.put(key, object.value());
});
this.broadcast = sparkContext.broadcast(toBroadcast);
}
代码示例来源:origin: apache/tinkerpop
@Override
public <R> R get(final String key) throws IllegalArgumentException {
if (!this.memoryComputeKeys.containsKey(key))
throw Memory.Exceptions.memoryDoesNotExist(key);
if (this.inExecute && !this.memoryComputeKeys.get(key).isBroadcast())
throw Memory.Exceptions.memoryDoesNotExist(key);
final ObjectWritable<R> r = (ObjectWritable<R>) (this.inExecute ? this.broadcast.value().get(key) : this.sparkMemory.get(key).value());
if (null == r || r.isEmpty())
throw Memory.Exceptions.memoryDoesNotExist(key);
else
return r.get();
}
代码示例来源:origin: org.apache.pig/pig
public T getValue() {
if (accumulator != null) {
return accumulator.value();
} else {
return null;
}
}
代码示例来源:origin: com.github.hyukjinkwon/spark-client
public long getValue() {
if (accumulator != null) {
return accumulator.value();
} else {
return accumValue;
}
}
代码示例来源:origin: org.apache.beam/beam-runners-spark
private static void checkpoint() throws IOException {
if (checkpointFilePath != null) {
Checkpoint.writeObject(fileSystem, checkpointFilePath, instance.value());
}
}
代码示例来源:origin: org.apache.beam/beam-runners-spark
private static void checkpoint() throws IOException {
if (checkpointFilePath != null) {
Checkpoint.writeObject(fileSystem, checkpointFilePath, instance.value());
}
}
代码示例来源:origin: ai.grakn/grakn-kb
protected void broadcastMemory(final JavaSparkContext sparkContext) {
this.broadcast.destroy(true); // do we need to block?
final Map<String, Object> toBroadcast = new HashMap<>();
this.sparkMemory.forEach((key, object) -> {
if (!object.value().isEmpty() && this.memoryComputeKeys.get(key).isBroadcast()) {
toBroadcast.put(key, object.value());
}
});
this.broadcast = sparkContext.broadcast(toBroadcast);
}
代码示例来源:origin: org.apache.tinkerpop/spark-gremlin
protected void broadcastMemory(final JavaSparkContext sparkContext) {
this.broadcast.destroy(true); // do we need to block?
final Map<String, Object> toBroadcast = new HashMap<>();
this.sparkMemory.forEach((key, object) -> {
if (!object.value().isEmpty() && this.memoryComputeKeys.get(key).isBroadcast())
toBroadcast.put(key, object.value());
});
this.broadcast = sparkContext.broadcast(toBroadcast);
}
代码示例来源:origin: apache/crunch
private Counters getCounters() {
Counters c = new Counters();
Map<String, Map<String, Long>> values = counters.value();
for (Map.Entry<String, Map<String, Long>> e : values.entrySet()) {
CounterGroup cg = c.getGroup(e.getKey());
for (Map.Entry<String, Long> f : e.getValue().entrySet()) {
cg.findCounter(f.getKey()).setValue(f.getValue());
}
}
return c;
}
代码示例来源:origin: org.qcri.rheem/rheem-spark
@Override
public OptionalLong getMeasuredCardinality() {
if (this.accumulator != null) {
this.setMeasuredCardinality(this.accumulator.value());
}
return super.getMeasuredCardinality();
}
代码示例来源:origin: org.apache.beam/beam-runners-spark
@Override
public MetricResults metrics() {
return asAttemptedOnlyMetricResults(MetricsAccumulator.getInstance().value());
}
代码示例来源:origin: org.qcri.rheem/rheem-spark
@Override
protected void doDispose() {
if (this.accumulator != null) {
this.setMeasuredCardinality(this.accumulator.value());
this.accumulator = null;
}
if (this.isRddCached() && this.rdd != null) {
Actions.doSafe(this.rdd::unpersist);
logger.debug("Unpersisted {}.", this.rdd);
this.rdd = null;
}
}
代码示例来源:origin: org.apache.tinkerpop/spark-gremlin
@Override
public <R> R get(final String key) throws IllegalArgumentException {
if (!this.memoryComputeKeys.containsKey(key))
throw Memory.Exceptions.memoryDoesNotExist(key);
if (this.inExecute && !this.memoryComputeKeys.get(key).isBroadcast())
throw Memory.Exceptions.memoryDoesNotExist(key);
final ObjectWritable<R> r = (ObjectWritable<R>) (this.inExecute ? this.broadcast.value().get(key) : this.sparkMemory.get(key).value());
if (null == r || r.isEmpty())
throw Memory.Exceptions.memoryDoesNotExist(key);
else
return r.get();
}
内容来源于网络,如有侵权,请联系作者删除!