org.apache.spark.broadcast.Broadcast类的使用及代码示例

x33g5p2x  于2022-01-16 转载在 其他  
字(8.4k)|赞(0)|评价(0)|浏览(245)

本文整理了Java中org.apache.spark.broadcast.Broadcast类的一些代码示例，展示了Broadcast类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Broadcast类的具体详情如下：
包路径：org.apache.spark.broadcast.Broadcast
类名称：Broadcast

Broadcast介绍

暂无

代码示例

代码示例来源：origin: OryxProject/oryx

@Override
 public Iterator<Tuple2<Integer,Integer>> call(
   Tuple2<Integer,Iterable<Integer>> userIDsAndItemIDs) {
  Integer userID = userIDsAndItemIDs._1();
  Collection<Integer> positiveItemIDs = Sets.newHashSet(userIDsAndItemIDs._2());
  int numPositive = positiveItemIDs.size();
  Collection<Tuple2<Integer,Integer>> negative = new ArrayList<>(numPositive);
  List<Integer> allItemIDs = allItemIDsBC.value();
  int numItems = allItemIDs.size();
  // Sample about as many negative examples as positive
  for (int i = 0; i < numItems && negative.size() < numPositive; i++) {
   Integer itemID = allItemIDs.get(random.nextInt(numItems));
   if (!positiveItemIDs.contains(itemID)) {
    negative.add(new Tuple2<>(userID, itemID));
   }
  }
  return negative.iterator();
 }
});

代码示例来源：origin: deeplearning4j/dl4j-examples

@Override
  public DataSet call(String s) throws Exception {
    //Here: take a String, and map the characters to a one-hot representation
    Map<Character, Integer> cti = ctiBroadcast.getValue();
    int length = s.length();
    INDArray features = Nd4j.zeros(1, N_CHARS, length - 1);
    INDArray labels = Nd4j.zeros(1, N_CHARS, length - 1);
    char[] chars = s.toCharArray();
    int[] f = new int[3];
    int[] l = new int[3];
    for (int i = 0; i < chars.length - 2; i++) {
      f[1] = cti.get(chars[i]);
      f[2] = i;
      l[1] = cti.get(chars[i + 1]);   //Predict the next character given past and current characters
      l[2] = i;
      features.putScalar(f, 1.0);
      labels.putScalar(l, 1.0);
    }
    return new DataSet(features, labels);
  }
}

代码示例来源：origin: OryxProject/oryx

bUserIDToIndex.unpersist();
bItemIDToIndex.unpersist();

代码示例来源：origin: SeldonIO/seldon-server

@Override
  public String call(Tuple2<String, ActionData> v1) throws Exception {
    Map<String, Integer> m = broadcastVar.getValue();
    ActionData actionData = v1._2;
    if (actionData.userid == 0) {
      String key = currentClient + "_" + actionData.client_userid;
      if (m.containsKey(key)) {
        actionData.userid = m.get(key);
      } else {
        return "";
      }
    }
    String json = JobUtils.getJsonFromActionData(actionData);
    return json;
  }
});

代码示例来源：origin: usc-isi-i2/Web-Karma

@Override
  public Iterable<Tuple2<String, String>> call(Tuple2<String, String> writableIterableTuple2) throws Exception {
    List<Tuple2<String, String>> results = new LinkedList<>();
    Properties karmaContentSettings = new Properties();
    for(Map.Entry<Object, Object> objectObjectEntry : karmaSettings.entrySet())
      karmaContentSettings.put(objectObjectEntry.getKey(), objectObjectEntry.getValue());
    karmaContentSettings.put("model.content", model.value());
    karmaContentSettings.put("context.content", context.getValue());
    
    if(outputFormat != null && outputFormat.equals("n3")) {
      final N3Impl mapper = new N3Impl(karmaContentSettings);
      String result = mapper.mapResult(writableIterableTuple2._1, writableIterableTuple2._2);
      String[] lines = result.split("(\r\n|\n)");
      for(String line : lines)
      {
        if((line = line.trim()).isEmpty())
        {
          continue;
        }
        int splitBetweenSubjectAndPredicate = line.indexOf(' ');
        String key = (line.substring(0, splitBetweenSubjectAndPredicate));
        String value = line;
        results.add(new Tuple2<>(key, value));
      }
    }
    return results;
  }
});

代码示例来源：origin: apache/tinkerpop

protected void broadcastMemory(final JavaSparkContext sparkContext) {
  this.broadcast.destroy(true); // do we need to block?
  final Map<String, Object> toBroadcast = new HashMap<>();
  this.sparkMemory.forEach((key, object) -> {
    if (!object.value().isEmpty() && this.memoryComputeKeys.get(key).isBroadcast())
      toBroadcast.put(key, object.value());
  });
  this.broadcast = sparkContext.broadcast(toBroadcast);
}

代码示例来源：origin: com.davidbracewell/mango

@Override
public synchronized void updateConfig() {
 if (configBroadcast != null && configBroadcast.isValid()) {
   configBroadcast.destroy();
 }
 configBroadcast = broadcast(Config.getInstance());
}

代码示例来源：origin: apache/tinkerpop

@Override
public Set<String> keys() {
  if (this.inExecute)
    return this.broadcast.getValue().keySet();
  else {
    final Set<String> trueKeys = new HashSet<>();
    this.sparkMemory.forEach((key, value) -> {
      if (!value.value().isEmpty())
        trueKeys.add(key);
    });
    return Collections.unmodifiableSet(trueKeys);
  }
}

代码示例来源：origin: usc-isi-i2/Web-Karma

objectObjectEntry.getKey(),
      objectObjectEntry.getValue());
karmaContentSettings.put("model.content", model.value());
karmaContentSettings.put("context.content",
    context.getValue());

代码示例来源：origin: ai.grakn/grakn-kb

protected void broadcastMemory(final JavaSparkContext sparkContext) {
  this.broadcast.destroy(true); // do we need to block?
  final Map<String, Object> toBroadcast = new HashMap<>();
  this.sparkMemory.forEach((key, object) -> {
    if (!object.value().isEmpty() && this.memoryComputeKeys.get(key).isBroadcast()) {
      toBroadcast.put(key, object.value());
    }
  });
  this.broadcast = sparkContext.broadcast(toBroadcast);
}

代码示例来源：origin: databricks/learning-spark

public Tuple2<String, Integer> call(Tuple2<String, Integer> callSignCount) {
   String sign = callSignCount._1();
   String country = lookupCountry(sign, signPrefixes.value());
   return new Tuple2(country, callSignCount._2());
  }}).reduceByKey(new SumInts());
countryContactCounts.saveAsTextFile(outputDir + "/countries.txt");

代码示例来源：origin: org.qcri.rheem/rheem-spark

@Override
@SuppressWarnings("unchecked")
public <T> Collection<T> getBroadcast(String name) {
  final Broadcast<?> broadcast = this.broadcasts.get(name);
  if (broadcast == null) {
    throw new RheemException("No such broadcast found: " + name);
  }
  return (Collection<T>) broadcast.getValue();
}

代码示例来源：origin: locationtech/geowave

final JavaSparkContext javaSC = JavaSparkContext.fromSparkContext(sc);
final NumericIndexStrategy leftStrategy = leftRDD.getIndexStrategy().getValue();
final NumericIndexStrategy rightStrategy = rightRDD.getIndexStrategy().getValue();
  rightTiers =
    prepareForReproject(
      rightIndex.filter(t -> higherBroadcast.value().contains(t._1().getBytes()[0])),
      largePartitionerCount);
  leftTiers =
    prepareForReproject(
      leftIndex.filter(t -> higherBroadcast.value().contains(t._1.getBytes()[0])),
      largePartitionerCount);

代码示例来源：origin: org.apache.tinkerpop/spark-gremlin

protected void broadcastMemory(final JavaSparkContext sparkContext) {
  this.broadcast.destroy(true); // do we need to block?
  final Map<String, Object> toBroadcast = new HashMap<>();
  this.sparkMemory.forEach((key, object) -> {
    if (!object.value().isEmpty() && this.memoryComputeKeys.get(key).isBroadcast())
      toBroadcast.put(key, object.value());
  });
  this.broadcast = sparkContext.broadcast(toBroadcast);
}

代码示例来源：origin: OryxProject/oryx

trainingRatingDataRDD.unpersist(false);
bUserIDToIndex.unpersist();
bItemIDToIndex.unpersist();
unpersist(model);
bUserIndexToID.unpersist();
bItemIndexToID.unpersist();

代码示例来源：origin: OryxProject/oryx

private static void saveFeaturesRDD(JavaPairRDD<Integer,float[]> features,
                  Path path,
                  Broadcast<? extends Map<Integer,String>> bIndexToID) {
 log.info("Saving features RDD to {}", path);
 features.map(keyAndVector -> {
  String id = bIndexToID.value().get(keyAndVector._1());
  float[] vector = keyAndVector._2();
  return TextUtils.joinJSON(Arrays.asList(id, vector));
 }).saveAsTextFile(path.toString(), GzipCodec.class);
}

代码示例来源：origin: org.apache.tinkerpop/spark-gremlin

@Override
public Set<String> keys() {
  if (this.inExecute)
    return this.broadcast.getValue().keySet();
  else {
    final Set<String> trueKeys = new HashSet<>();
    this.sparkMemory.forEach((key, value) -> {
      if (!value.value().isEmpty())
        trueKeys.add(key);
    });
    return Collections.unmodifiableSet(trueKeys);
  }
}

代码示例来源：origin: cerner/bunsen

/**
 * Pops a BroadcastableValueSets from the user-defined function stack.
 *
 * @param spark the spark session
 * @return true if there is still a registered in_valuset UDF, false otherwise
 */
public static synchronized boolean popUdf(SparkSession spark) {
 if (valueSetStack.isEmpty()) {
  return false;
 } else {
  // Cleanup the previously broadcast valuesets
  Broadcast<BroadcastableValueSets> old = valueSetStack.pop();
  old.destroy();
  if (valueSetStack.isEmpty()) {
   return false;
  } else {
   // Re-apply the previous function.
   Broadcast<BroadcastableValueSets> current = valueSetStack.peek();
   spark.udf()
     .register("in_valueset",
       new InValuesetUdf(current),
       DataTypes.BooleanType);
   return true;
  }
 }
}

代码示例来源：origin: org.apache.beam/beam-runners-spark

public void unpersist() {
 this.bcast.unpersist();
}

代码示例来源：origin: OryxProject/oryx

return new Tuple2<>(
  Long.valueOf(tokens[3]),
  new Rating(bUserIDToIndex.value().get(tokens[0]),
        bItemIDToIndex.value().get(tokens[1]),

内容来源于网络，如有侵权，请联系作者删除！

Broadcast

相关文章

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com