org.apache.spark.sql.Row.get()方法的使用及代码示例

x33g5p2x  于2022-01-28 转载在 其他  
字(12.0k)|赞(0)|评价(0)|浏览(273)

本文整理了Java中org.apache.spark.sql.Row.get方法的一些代码示例，展示了Row.get的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Row.get方法的具体详情如下：
包路径：org.apache.spark.sql.Row
类名称：Row
方法名：get

Row.get介绍

暂无

代码示例

代码示例来源：origin: apache/kylin

@Override
  public String[] call(Row row) throws Exception {
    String[] result = new String[row.size()];
    for (int i = 0; i < row.size(); i++) {
      final Object o = row.get(i);
      if (o != null) {
        result[i] = o.toString();
      } else {
        result[i] = null;
      }
    }
    return result;
  }
});

代码示例来源：origin: org.apache.spark/spark-sql_2.11

@Test
 public void testUDF() {
  UserDefinedFunction foo = udf((Integer i, String s) -> i.toString() + s, DataTypes.StringType);
  Dataset<Row> df = spark.table("testData").select(foo.apply(col("key"), col("value")));
  String[] result = df.collectAsList().stream().map(row -> row.getString(0))
   .toArray(String[]::new);
  String[] expected = spark.table("testData").collectAsList().stream()
   .map(row -> row.get(0).toString() + row.getString(1)).toArray(String[]::new);
  Assert.assertArrayEquals(expected, result);
 }
}

代码示例来源：origin: org.apache.spark/spark-sql_2.10

private static void appendValue(ColumnVector dst, DataType t, Row src, int fieldIdx) {
 if (t instanceof ArrayType) {
  ArrayType at = (ArrayType)t;
  if (src.isNullAt(fieldIdx)) {
   dst.appendNull();
  } else {
   List<Object> values = src.getList(fieldIdx);
   dst.appendArray(values.size());
   for (Object o : values) {
    appendValue(dst.arrayData(), at.elementType(), o);
   }
  }
 } else if (t instanceof StructType) {
  StructType st = (StructType)t;
  if (src.isNullAt(fieldIdx)) {
   dst.appendStruct(true);
  } else {
   dst.appendStruct(false);
   Row c = src.getStruct(fieldIdx);
   for (int i = 0; i < st.fields().length; i++) {
    appendValue(dst.getChildColumn(i), st.fields()[i].dataType(), c, i);
   }
  }
 } else {
  appendValue(dst, t, src.get(fieldIdx));
 }
}

代码示例来源：origin: org.apache.spark/spark-sql_2.10

@Test
public void testCrosstab() {
 Dataset<Row> df = spark.table("testData2");
 Dataset<Row> crosstab = df.stat().crosstab("a", "b");
 String[] columnNames = crosstab.schema().fieldNames();
 Assert.assertEquals("a_b", columnNames[0]);
 Assert.assertEquals("1", columnNames[1]);
 Assert.assertEquals("2", columnNames[2]);
 List<Row> rows = crosstab.collectAsList();
 rows.sort(crosstabRowComparator);
 Integer count = 1;
 for (Row row : rows) {
  Assert.assertEquals(row.get(0).toString(), count.toString());
  Assert.assertEquals(1L, row.getLong(1));
  Assert.assertEquals(1L, row.getLong(2));
  count++;
 }
}

代码示例来源：origin: org.apache.spark/spark-sql

@Test
 public void testUDF() {
  UserDefinedFunction foo = udf((Integer i, String s) -> i.toString() + s, DataTypes.StringType);
  Dataset<Row> df = spark.table("testData").select(foo.apply(col("key"), col("value")));
  String[] result = df.collectAsList().stream().map(row -> row.getString(0))
   .toArray(String[]::new);
  String[] expected = spark.table("testData").collectAsList().stream()
   .map(row -> row.get(0).toString() + row.getString(1)).toArray(String[]::new);
  Assert.assertArrayEquals(expected, result);
 }
}

代码示例来源：origin: databricks/learning-spark

Row[] result = topTweets.collect();
for (Row row : result) {
 System.out.println(row.get(0));
Row[] lengths = tweetLength.collect();
for (Row row : result) {
 System.out.println(row.get(0));

代码示例来源：origin: org.apache.spark/spark-sql_2.11

@Test
public void testCrosstab() {
 Dataset<Row> df = spark.table("testData2");
 Dataset<Row> crosstab = df.stat().crosstab("a", "b");
 String[] columnNames = crosstab.schema().fieldNames();
 Assert.assertEquals("a_b", columnNames[0]);
 Assert.assertEquals("1", columnNames[1]);
 Assert.assertEquals("2", columnNames[2]);
 List<Row> rows = crosstab.collectAsList();
 rows.sort(crosstabRowComparator);
 Integer count = 1;
 for (Row row : rows) {
  Assert.assertEquals(row.get(0).toString(), count.toString());
  Assert.assertEquals(1L, row.getLong(1));
  Assert.assertEquals(1L, row.getLong(2));
  count++;
 }
}

代码示例来源：origin: org.apache.spark/spark-sql_2.11

private static void appendValue(WritableColumnVector dst, DataType t, Row src, int fieldIdx) {
 if (t instanceof ArrayType) {
  ArrayType at = (ArrayType)t;
  if (src.isNullAt(fieldIdx)) {
   dst.appendNull();
  } else {
   List<Object> values = src.getList(fieldIdx);
   dst.appendArray(values.size());
   for (Object o : values) {
    appendValue(dst.arrayData(), at.elementType(), o);
   }
  }
 } else if (t instanceof StructType) {
  StructType st = (StructType)t;
  if (src.isNullAt(fieldIdx)) {
   dst.appendStruct(true);
  } else {
   dst.appendStruct(false);
   Row c = src.getStruct(fieldIdx);
   for (int i = 0; i < st.fields().length; i++) {
    appendValue(dst.getChild(i), st.fields()[i].dataType(), c, i);
   }
  }
 } else {
  appendValue(dst, t, src.get(fieldIdx));
 }
}

代码示例来源：origin: org.apache.spark/spark-sql_2.10

@Test
public void dataFrameRDDOperations() {
 List<Person> personList = new ArrayList<>(2);
 Person person1 = new Person();
 person1.setName("Michael");
 person1.setAge(29);
 personList.add(person1);
 Person person2 = new Person();
 person2.setName("Yin");
 person2.setAge(28);
 personList.add(person2);
 JavaRDD<Row> rowRDD = jsc.parallelize(personList).map(
   person -> RowFactory.create(person.getName(), person.getAge()));
 List<StructField> fields = new ArrayList<>(2);
 fields.add(DataTypes.createStructField("", DataTypes.StringType, false));
 fields.add(DataTypes.createStructField("age", DataTypes.IntegerType, false));
 StructType schema = DataTypes.createStructType(fields);
 Dataset<Row> df = spark.createDataFrame(rowRDD, schema);
 df.createOrReplaceTempView("people");
 List<String> actual = spark.sql("SELECT * FROM people").toJavaRDD()
  .map(row -> row.getString(0) + "_" + row.get(1)).collect();
 List<String> expected = new ArrayList<>(2);
 expected.add("Michael_29");
 expected.add("Yin_28");
 Assert.assertEquals(expected, actual);
}

代码示例来源：origin: apache/phoenix

@Override
public Object getObject(int columnIndex) throws SQLException {
  return getCurrentRow().get(columnIndex-1);
}

代码示例来源：origin: org.apache.spark/spark-sql

@Test
public void testCrosstab() {
 Dataset<Row> df = spark.table("testData2");
 Dataset<Row> crosstab = df.stat().crosstab("a", "b");
 String[] columnNames = crosstab.schema().fieldNames();
 Assert.assertEquals("a_b", columnNames[0]);
 Assert.assertEquals("1", columnNames[1]);
 Assert.assertEquals("2", columnNames[2]);
 List<Row> rows = crosstab.collectAsList();
 rows.sort(crosstabRowComparator);
 Integer count = 1;
 for (Row row : rows) {
  Assert.assertEquals(row.get(0).toString(), count.toString());
  Assert.assertEquals(1L, row.getLong(1));
  Assert.assertEquals(1L, row.getLong(2));
  count++;
 }
}

代码示例来源：origin: org.apache.spark/spark-sql

private static void appendValue(WritableColumnVector dst, DataType t, Row src, int fieldIdx) {
 if (t instanceof ArrayType) {
  ArrayType at = (ArrayType)t;
  if (src.isNullAt(fieldIdx)) {
   dst.appendNull();
  } else {
   List<Object> values = src.getList(fieldIdx);
   dst.appendArray(values.size());
   for (Object o : values) {
    appendValue(dst.arrayData(), at.elementType(), o);
   }
  }
 } else if (t instanceof StructType) {
  StructType st = (StructType)t;
  if (src.isNullAt(fieldIdx)) {
   dst.appendStruct(true);
  } else {
   dst.appendStruct(false);
   Row c = src.getStruct(fieldIdx);
   for (int i = 0; i < st.fields().length; i++) {
    appendValue(dst.getChild(i), st.fields()[i].dataType(), c, i);
   }
  }
 } else {
  appendValue(dst, t, src.get(fieldIdx));
 }
}

代码示例来源：origin: org.apache.spark/spark-sql

@Test
public void dataFrameRDDOperations() {
 List<Person> personList = new ArrayList<>(2);
 Person person1 = new Person();
 person1.setName("Michael");
 person1.setAge(29);
 personList.add(person1);
 Person person2 = new Person();
 person2.setName("Yin");
 person2.setAge(28);
 personList.add(person2);
 JavaRDD<Row> rowRDD = jsc.parallelize(personList).map(
   person -> RowFactory.create(person.getName(), person.getAge()));
 List<StructField> fields = new ArrayList<>(2);
 fields.add(DataTypes.createStructField("", DataTypes.StringType, false));
 fields.add(DataTypes.createStructField("age", DataTypes.IntegerType, false));
 StructType schema = DataTypes.createStructType(fields);
 Dataset<Row> df = spark.createDataFrame(rowRDD, schema);
 df.createOrReplaceTempView("people");
 List<String> actual = spark.sql("SELECT * FROM people").toJavaRDD()
  .map(row -> row.getString(0) + "_" + row.get(1)).collect();
 List<String> expected = new ArrayList<>(2);
 expected.add("Michael_29");
 expected.add("Yin_28");
 Assert.assertEquals(expected, actual);
}

代码示例来源：origin: org.apache.spark/spark-sql_2.11

complexMap,
 null);
Assert.assertEquals(simpleStringArray, complexStruct.get(0));
Assert.assertEquals(simpleMap, complexStruct.get(1));
Assert.assertEquals(simpleStruct, complexStruct.get(2));
Assert.assertEquals(arrayOfMaps, complexStruct.get(3));
Assert.assertEquals(arrayOfRows, complexStruct.get(4));
Assert.assertEquals(complexMap, complexStruct.get(5));
Assert.assertNull(complexStruct.get(6));
Assert.assertEquals(arrayOfMaps, complexRow.get(0));
Assert.assertEquals(arrayOfRows, complexRow.get(1));
Assert.assertEquals(complexMap, complexRow.get(2));
Assert.assertEquals(complexStruct, complexRow.get(3));

代码示例来源：origin: org.apache.spark/spark-sql_2.11

@Test
public void dataFrameRDDOperations() {
 List<Person> personList = new ArrayList<>(2);
 Person person1 = new Person();
 person1.setName("Michael");
 person1.setAge(29);
 personList.add(person1);
 Person person2 = new Person();
 person2.setName("Yin");
 person2.setAge(28);
 personList.add(person2);
 JavaRDD<Row> rowRDD = jsc.parallelize(personList).map(
   person -> RowFactory.create(person.getName(), person.getAge()));
 List<StructField> fields = new ArrayList<>(2);
 fields.add(DataTypes.createStructField("", DataTypes.StringType, false));
 fields.add(DataTypes.createStructField("age", DataTypes.IntegerType, false));
 StructType schema = DataTypes.createStructType(fields);
 Dataset<Row> df = spark.createDataFrame(rowRDD, schema);
 df.createOrReplaceTempView("people");
 List<String> actual = spark.sql("SELECT * FROM people").toJavaRDD()
  .map(row -> row.getString(0) + "_" + row.get(1)).collect();
 List<String> expected = new ArrayList<>(2);
 expected.add("Michael_29");
 expected.add("Yin_28");
 Assert.assertEquals(expected, actual);
}

代码示例来源：origin: org.apache.spark/spark-sql_2.10

complexMap,
 null);
Assert.assertEquals(simpleStringArray, complexStruct.get(0));
Assert.assertEquals(simpleMap, complexStruct.get(1));
Assert.assertEquals(simpleStruct, complexStruct.get(2));
Assert.assertEquals(arrayOfMaps, complexStruct.get(3));
Assert.assertEquals(arrayOfRows, complexStruct.get(4));
Assert.assertEquals(complexMap, complexStruct.get(5));
Assert.assertNull(complexStruct.get(6));
Assert.assertEquals(arrayOfMaps, complexRow.get(0));
Assert.assertEquals(arrayOfRows, complexRow.get(1));
Assert.assertEquals(complexMap, complexRow.get(2));
Assert.assertEquals(complexStruct, complexRow.get(3));

代码示例来源：origin: org.apache.spark/spark-sql

complexMap,
 null);
Assert.assertEquals(simpleStringArray, complexStruct.get(0));
Assert.assertEquals(simpleMap, complexStruct.get(1));
Assert.assertEquals(simpleStruct, complexStruct.get(2));
Assert.assertEquals(arrayOfMaps, complexStruct.get(3));
Assert.assertEquals(arrayOfRows, complexStruct.get(4));
Assert.assertEquals(complexMap, complexStruct.get(5));
Assert.assertNull(complexStruct.get(6));
Assert.assertEquals(arrayOfMaps, complexRow.get(0));
Assert.assertEquals(arrayOfRows, complexRow.get(1));
Assert.assertEquals(complexMap, complexRow.get(2));
Assert.assertEquals(complexStruct, complexRow.get(3));

代码示例来源：origin: org.apache.spark/spark-sql

@Test
public void testExecution() {
 Dataset<Row> df = spark.table("testData").filter("key = 1");
 Assert.assertEquals(1, df.select("key").collectAsList().get(0).get(0));
}

代码示例来源：origin: org.apache.spark/spark-sql_2.10

@Test
public void testExecution() {
 Dataset<Row> df = spark.table("testData").filter("key = 1");
 Assert.assertEquals(1, df.select("key").collectAsList().get(0).get(0));
}

代码示例来源：origin: org.apache.spark/spark-sql_2.11

@Test
public void testExecution() {
 Dataset<Row> df = spark.table("testData").filter("key = 1");
 Assert.assertEquals(1, df.select("key").collectAsList().get(0).get(0));
}

内容来源于网络，如有侵权，请联系作者删除！

row

相关文章

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com