org.apache.spark.Accumulator类的使用及代码示例

x33g5p2x  于2022-01-16 转载在 其他  
字(7.8k)|赞(0)|评价(0)|浏览(108)

本文整理了Java中org.apache.spark.Accumulator类的一些代码示例，展示了Accumulator类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Accumulator类的具体详情如下：
包路径：org.apache.spark.Accumulator
类名称：Accumulator

Accumulator介绍

暂无

代码示例

代码示例来源：origin: apache/hive

public long getValue() {
 if (accumulator != null) {
  return accumulator.value();
 } else {
  return accumValue;
 }
}

代码示例来源：origin: apache/hive

public void increment(long incr) {
 accumulator.add(incr);
}

代码示例来源：origin: org.apache.spark/spark-core

assertEquals((Integer) 25, intAccum.value());
rdd.foreach(x -> doubleAccum.add((double) x));
assertEquals((Double) 25.0, doubleAccum.value());
rdd.foreach(x -> floatAccum.add((float) x));
assertEquals((Float) 25.0f, floatAccum.value());
floatAccum.setValue(5.0f);
assertEquals((Float) 5.0f, floatAccum.value());

代码示例来源：origin: uber/hudi

public static int handleErrors(JavaSparkContext jsc, String instantTime, JavaRDD<WriteStatus> writeResponse) {
 Accumulator<Integer> errors = jsc.accumulator(0);
 writeResponse.foreach(writeStatus -> {
  if (writeStatus.hasErrors()) {
   errors.add(1);
   logger.error(String.format("Error processing records :writeStatus:%s",
     writeStatus.getStat().toString()));
  }
 });
 if (errors.value() == 0) {
  logger.info(
    String.format("Dataset imported into hoodie dataset with %s instant time.", instantTime));
  return 0;
 }
 logger.error(String.format("Import failed with %d errors.", errors.value()));
 return -1;
}

代码示例来源：origin: org.apache.beam/beam-runners-spark

/** Init aggregators accumulator if it has not been initiated. This method is idempotent. */
public static void init(SparkPipelineOptions opts, JavaSparkContext jsc) {
 if (instance == null) {
  synchronized (AggregatorsAccumulator.class) {
   if (instance == null) {
    Optional<CheckpointDir> maybeCheckpointDir =
      opts.isStreaming()
        ? Optional.of(new CheckpointDir(opts.getCheckpointDir()))
        : Optional.absent();
    Accumulator<NamedAggregators> accumulator =
      jsc.sc().accumulator(new NamedAggregators(), ACCUMULATOR_NAME, new AggAccumParam());
    if (maybeCheckpointDir.isPresent()) {
     Optional<NamedAggregators> maybeRecoveredValue =
       recoverValueFromCheckpoint(jsc, maybeCheckpointDir.get());
     if (maybeRecoveredValue.isPresent()) {
      accumulator.setValue(maybeRecoveredValue.get());
     }
    }
    instance = accumulator;
   }
  }
  LOG.info("Instantiated aggregators accumulator: " + instance.value());
 }
}

代码示例来源：origin: apache/tinkerpop

@Override
public void set(final String key, final Object value) {
  checkKeyValue(key, value);
  if (this.inExecute)
    throw Memory.Exceptions.memorySetOnlyDuringVertexProgramSetUpAndTerminate(key);
  else
    this.sparkMemory.get(key).setValue(new ObjectWritable<>(value));
}

代码示例来源：origin: org.apache.beam/beam-runners-spark

private MetricsContainer metricsContainer() {
  return metricsAccum.localValue().getContainer(stepName);
 }
}

代码示例来源：origin: com.uber.hoodie/hoodie-utilities

public static int handleErrors(JavaSparkContext jsc, String instantTime, JavaRDD<WriteStatus> writeResponse) {
 Accumulator<Integer> errors = jsc.accumulator(0);
 writeResponse.foreach(writeStatus -> {
  if (writeStatus.hasErrors()) {
   errors.add(1);
   logger.error(String.format("Error processing records :writeStatus:%s",
     writeStatus.getStat().toString()));
  }
 });
 if (errors.value() == 0) {
  logger.info(
    String.format("Dataset imported into hoodie dataset with %s instant time.", instantTime));
  return 0;
 }
 logger.error(String.format("Import failed with %d errors.", errors.value()));
 return -1;
}

代码示例来源：origin: org.apache.beam/beam-runners-spark

/** Init metrics accumulator if it has not been initiated. This method is idempotent. */
public static void init(SparkPipelineOptions opts, JavaSparkContext jsc) {
 if (instance == null) {
  synchronized (MetricsAccumulator.class) {
   if (instance == null) {
    Optional<CheckpointDir> maybeCheckpointDir =
      opts.isStreaming()
        ? Optional.of(new CheckpointDir(opts.getCheckpointDir()))
        : Optional.absent();
    Accumulator<MetricsContainerStepMap> accumulator =
      jsc.sc()
        .accumulator(
          new SparkMetricsContainerStepMap(),
          ACCUMULATOR_NAME,
          new MetricsAccumulatorParam());
    if (maybeCheckpointDir.isPresent()) {
     Optional<MetricsContainerStepMap> maybeRecoveredValue =
       recoverValueFromCheckpoint(jsc, maybeCheckpointDir.get());
     if (maybeRecoveredValue.isPresent()) {
      accumulator.setValue(maybeRecoveredValue.get());
     }
    }
    instance = accumulator;
   }
  }
  LOG.info("Instantiated metrics accumulator: " + instance.value());
 }
}

代码示例来源：origin: org.apache.tinkerpop/spark-gremlin

@Override
public void set(final String key, final Object value) {
  checkKeyValue(key, value);
  if (this.inExecute)
    throw Memory.Exceptions.memorySetOnlyDuringVertexProgramSetUpAndTerminate(key);
  else
    this.sparkMemory.get(key).setValue(new ObjectWritable<>(value));
}

代码示例来源：origin: org.apache.beam/beam-runners-spark

@Override
public scala.collection.Iterator<WindowedValue<T>> compute(
  final Partition split, final TaskContext context) {
 final MetricsContainer metricsContainer = metricsAccum.localValue().getContainer(stepName);
 @SuppressWarnings("unchecked")
 final BoundedSource.BoundedReader<T> reader = createReader((SourcePartition<T>) split);
 final Iterator<WindowedValue<T>> readerIterator =
   new ReaderToIteratorAdapter<>(metricsContainer, reader);
 return new InterruptibleIterator<>(context, JavaConversions.asScalaIterator(readerIterator));
}

代码示例来源：origin: apache/hive

SparkCounter snapshot() {
 return new SparkCounter(name, displayName, accumulator.value());
}

代码示例来源：origin: databricks/learning-spark

rdd.foreach(new VoidFunction<String>(){ public void call(String line) {
   if (line.contains("KK6JKQ")) {
    count.add(1);
   }
  }});
System.out.println("Lines with 'KK6JKQ': " + count.value());

代码示例来源：origin: org.apache.spark/spark-core_2.10

assertEquals((Integer) 25, intAccum.value());
rdd.foreach(x -> doubleAccum.add((double) x));
assertEquals((Double) 25.0, doubleAccum.value());
rdd.foreach(x -> floatAccum.add((float) x));
assertEquals((Float) 25.0f, floatAccum.value());
floatAccum.setValue(5.0f);
assertEquals((Float) 5.0f, floatAccum.value());

代码示例来源：origin: ai.grakn/grakn-kb

@Override
public void set(final String key, final Object value) {
  checkKeyValue(key, value);
  if (this.inExecute) {
    throw Memory.Exceptions.memorySetOnlyDuringVertexProgramSetUpAndTerminate(key);
  } else {
    this.sparkMemory.get(key).setValue(new ObjectWritable<>(value));
  }
}

代码示例来源：origin: databricks/learning-spark

System.out.println("Lines with 'KK6JKQ': " + count.value());
  }});
callSigns.saveAsTextFile(outputDir + "/callsigns");
System.out.println("Blank lines: "+ blankLines.value());
if (invalidSignCount.value() < 0.1 * validSignCount.value()) {
 contactCounts.saveAsTextFile(outputDir + "/contactCount");
} else {
 System.out.println("Too many errors " + invalidSignCount.value() +
           " for " + validSignCount.value());
 System.exit(1);

代码示例来源：origin: databricks/learning-spark

new Function<String, Boolean>(){ public Boolean call(String callSign) {
  Pattern p = Pattern.compile("\\A\\d?\\p{Alpha}{1,2}\\d{1,4}\\p{Alpha}{1,3}\\Z");
  Matcher m = p.matcher(callSign);
  boolean b = m.matches();
  if (b) {
   validSignCount.add(1);
  } else {
   invalidSignCount.add(1);
  }
  return b;
 }
});

代码示例来源：origin: org.apache.spark/spark-core_2.11

assertEquals((Integer) 25, intAccum.value());
rdd.foreach(x -> doubleAccum.add((double) x));
assertEquals((Double) 25.0, doubleAccum.value());
rdd.foreach(x -> floatAccum.add((float) x));
assertEquals((Float) 25.0f, floatAccum.value());
floatAccum.setValue(5.0f);
assertEquals((Float) 5.0f, floatAccum.value());

代码示例来源：origin: apache/tinkerpop

@Override
public Set<String> keys() {
  if (this.inExecute)
    return this.broadcast.getValue().keySet();
  else {
    final Set<String> trueKeys = new HashSet<>();
    this.sparkMemory.forEach((key, value) -> {
      if (!value.value().isEmpty())
        trueKeys.add(key);
    });
    return Collections.unmodifiableSet(trueKeys);
  }
}

代码示例来源：origin: databricks/learning-spark

new FlatMapFunction<String, String>() { public Iterable<String> call(String line) {
   if (line.equals("")) {
    blankLines.add(1);
   }
   return Arrays.asList(line.split(" "));
  }});
callSigns.saveAsTextFile(outputDir + "/callsigns");

内容来源于网络，如有侵权，请联系作者删除！

Accumulator

相关文章

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com