org.apache.spark.rdd.RDD.persist()方法的使用及代码示例

x33g5p2x  于2022-01-28 转载在 其他  
字(5.0k)|赞(0)|评价(0)|浏览(225)

本文整理了Java中org.apache.spark.rdd.RDD.persist方法的一些代码示例,展示了RDD.persist的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。RDD.persist方法的具体详情如下:
包路径:org.apache.spark.rdd.RDD
类名称:RDD
方法名:persist

RDD.persist介绍

暂无

代码示例

代码示例来源:origin: OryxProject/oryx

private static RDD<Tuple2<Object,double[]>> readAndConvertFeatureRDD(
  JavaPairRDD<String,float[]> javaRDD,
  Broadcast<? extends Map<String,Integer>> bIdToIndex) {
 RDD<Tuple2<Integer,double[]>> scalaRDD = javaRDD.mapToPair(t ->
   new Tuple2<>(bIdToIndex.value().get(t._1()), t._2())
 ).mapValues(f -> {
   double[] d = new double[f.length];
   for (int i = 0; i < d.length; i++) {
    d[i] = f[i];
   }
   return d;
  }
 ).rdd();
 // This mimics the persistence level establish by ALS training methods
 scalaRDD.persist(StorageLevel.MEMORY_AND_DISK());
 @SuppressWarnings("unchecked")
 RDD<Tuple2<Object,double[]>> objKeyRDD = (RDD<Tuple2<Object,double[]>>) (RDD<?>) scalaRDD;
 return objKeyRDD;
}

代码示例来源:origin: apache/crunch

public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) {
 if (!runtime.isValid(rdd)) {
  rdd = getJavaRDDLikeInternal(runtime);
  rdd.rdd().setName(getName());
  StorageLevel sl = runtime.getStorageLevel(this);
  if (sl != null) {
   rdd.rdd().persist(sl);
  }
 }
 return rdd;
}

代码示例来源:origin: org.apache.crunch/crunch-spark

@Override
public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) {
 if (!runtime.isValid(rdd)) {
  rdd = getJavaRDDLikeInternal(runtime);
  rdd.rdd().setName(getName());
  StorageLevel sl = runtime.getStorageLevel(this);
  if (sl != null) {
   rdd.rdd().persist(sl);
  }
 }
 return rdd;
}

代码示例来源:origin: org.apache.crunch/crunch-spark

public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) {
 if (!runtime.isValid(rdd)) {
  rdd = getJavaRDDLikeInternal(runtime);
  rdd.rdd().setName(getName());
  StorageLevel sl = runtime.getStorageLevel(this);
  if (sl != null) {
   rdd.rdd().persist(sl);
  }
 }
 return rdd;
}

代码示例来源:origin: org.apache.crunch/crunch-spark

public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) {
 if (!runtime.isValid(rdd)) {
  rdd = getJavaRDDLikeInternal(runtime);
  rdd.rdd().setName(getName());
  StorageLevel sl = runtime.getStorageLevel(this);
  if (sl != null) {
   rdd.rdd().persist(sl);
  }
 }
 return rdd;
}

代码示例来源:origin: org.apache.crunch/crunch-spark

@Override
public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) {
 if (!runtime.isValid(rdd)) {
  rdd = getJavaRDDLikeInternal(runtime);
  rdd.rdd().setName(getName());
  StorageLevel sl = runtime.getStorageLevel(this);
  if (sl != null) {
   rdd.rdd().persist(sl);
  }
 }
 return rdd;
}

代码示例来源:origin: apache/crunch

public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) {
 if (!runtime.isValid(rdd)) {
  rdd = getJavaRDDLikeInternal(runtime);
  rdd.rdd().setName(getName());
  StorageLevel sl = runtime.getStorageLevel(this);
  if (sl != null) {
   rdd.rdd().persist(sl);
  }
 }
 return rdd;
}

代码示例来源:origin: org.apache.crunch/crunch-spark

public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) {
 if (!runtime.isValid(rdd)) {
  rdd = getJavaRDDLikeInternal(runtime);
  rdd.rdd().setName(getName());
  StorageLevel sl = runtime.getStorageLevel(this);
  if (sl != null) {
   rdd.rdd().persist(sl);
  }
 }
 return rdd;
}

代码示例来源:origin: org.apache.crunch/crunch-spark

public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) {
 if (!runtime.isValid(rdd)) {
  rdd = getJavaRDDLikeInternal(runtime);
  rdd.rdd().setName(getName());
  StorageLevel sl = runtime.getStorageLevel(this);
  if (sl != null) {
   rdd.rdd().persist(sl);
  }
 }
 return rdd;
}

代码示例来源:origin: apache/crunch

public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) {
 if (!runtime.isValid(rdd)) {
  rdd = getJavaRDDLikeInternal(runtime);
  rdd.rdd().setName(getName());
  StorageLevel sl = runtime.getStorageLevel(this);
  if (sl != null) {
   rdd.rdd().persist(sl);
  }
 }
 return rdd;
}

代码示例来源:origin: apache/crunch

@Override
public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) {
 if (!runtime.isValid(rdd)) {
  rdd = getJavaRDDLikeInternal(runtime);
  rdd.rdd().setName(getName());
  StorageLevel sl = runtime.getStorageLevel(this);
  if (sl != null) {
   rdd.rdd().persist(sl);
  }
 }
 return rdd;
}

代码示例来源:origin: apache/crunch

public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) {
 if (!runtime.isValid(rdd)) {
  rdd = getJavaRDDLikeInternal(runtime);
  rdd.rdd().setName(getName());
  StorageLevel sl = runtime.getStorageLevel(this);
  if (sl != null) {
   rdd.rdd().persist(sl);
  }
 }
 return rdd;
}

代码示例来源:origin: apache/crunch

@Override
public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) {
 if (!runtime.isValid(rdd)) {
  rdd = getJavaRDDLikeInternal(runtime);
  rdd.rdd().setName(getName());
  StorageLevel sl = runtime.getStorageLevel(this);
  if (sl != null) {
   rdd.rdd().persist(sl);
  }
 }
 return rdd;
}

代码示例来源:origin: apache/crunch

public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) {
 if (!runtime.isValid(rdd)) {
  rdd = getJavaRDDLikeInternal(runtime, runtime.getCombineFn());
  rdd.rdd().setName(getName());
  StorageLevel sl = runtime.getStorageLevel(this);
  if (sl != null) {
   rdd.rdd().persist(sl);
  }
 }
 return rdd;
}

代码示例来源:origin: org.apache.crunch/crunch-spark

public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) {
 if (!runtime.isValid(rdd)) {
  rdd = getJavaRDDLikeInternal(runtime, runtime.getCombineFn());
  rdd.rdd().setName(getName());
  StorageLevel sl = runtime.getStorageLevel(this);
  if (sl != null) {
   rdd.rdd().persist(sl);
  }
 }
 return rdd;
}

相关文章