cz.seznam.euphoria.core.executor.Executor类的使用及代码示例

x33g5p2x  于2022-01-19 转载在 其他  
字(5.6k)|赞(0)|评价(0)|浏览(246)

本文整理了Java中cz.seznam.euphoria.core.executor.Executor类的一些代码示例,展示了Executor类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Executor类的具体详情如下:
包路径:cz.seznam.euphoria.core.executor.Executor
类名称:Executor

Executor介绍

[英]The client side, public, technology independent interface to an executor.
[中]客户端,公共的,独立于技术的执行者接口。

代码示例

代码示例来源:origin: seznam/euphoria

  1. public void execute(Executor exec) throws Exception {
  2. exec.submit(this.wrap.getFlow()).get();
  3. }
  4. }

代码示例来源:origin: seznam/euphoria

  1. private Executor.Result execute(Flow flow) {
  2. DAG<Operator<?, ?>> dag = FlowUnfolder.unfold(flow, Executor.getBasicOps());

代码示例来源:origin: seznam/euphoria

  1. @Override
  2. public void shutdown() throws Exception {
  3. executor.shutdown();
  4. }
  5. };

代码示例来源:origin: seznam/euphoria

  1. executor.setAccumulatorProvider(accs);
  2. executor.submit(flow).get();
  3. } catch (InterruptedException | ExecutionException e) {
  4. throw new RuntimeException("Test failure at run #" + i, e);

代码示例来源:origin: seznam/euphoria

  1. executor.setAccumulatorProvider(accs);
  2. executor.submit(flow).get();
  3. } catch (InterruptedException | ExecutionException e) {
  4. throw new RuntimeException("Test failure at run #" + i, e);

代码示例来源:origin: seznam/euphoria

  1. public void execute(Executor executor) {
  2. final Flow flow = Flow.create("test");
  3. final ListDataSink<OUT> sink = ListDataSink.get();
  4. buildFlow(flow).persist(sink);
  5. executor.submit(flow).join();
  6. DatasetAssert.unorderedEquals(getOutput(), sink.getOutputs());
  7. }
  8. }

代码示例来源:origin: seznam/euphoria

  1. private Executor.Result execute(Flow flow) {
  2. DAG<Operator<?, ?>> dag = FlowUnfolder.unfold(flow, Executor.getBasicOps());

代码示例来源:origin: seznam/euphoria

  1. public void execute(Executor executor) {
  2. final Flow flow = Flow.create("test");
  3. final ListDataSink<OUT> sink = ListDataSink.get();
  4. buildFlow(flow).persist(sink);
  5. executor.submit(flow).join();
  6. DatasetAssert.unorderedEquals(getOutput(), sink.getOutputs());
  7. }
  8. }

代码示例来源:origin: seznam/euphoria

  1. @Test
  2. @SuppressWarnings("unchecked")
  3. public void testUnfoldBasic() {
  4. DAG<Operator<?, ?>> unfolded = FlowUnfolder.unfold(flow, Executor.getBasicOps());

代码示例来源:origin: seznam/euphoria

  1. executor.submit(flow).get();
  2. } catch (InterruptedException ex) {
  3. LOG.warn("Interrupted while waiting for the flow to finish.", ex);

代码示例来源:origin: seznam/euphoria

  1. @Test(expected = IllegalArgumentException.class)
  2. public void testMultipleOutputsToSameSink() throws Exception {
  3. flow = Flow.create(getClass().getSimpleName());
  4. input = flow.createInput(new MockStreamDataSource<>());
  5. Dataset<Object> mapped = MapElements.of(input).using(e -> e).output();
  6. Dataset<Pair<Object, Long>> reduced = ReduceByKey
  7. .of(mapped)
  8. .keyBy(e -> e).reduceBy(values -> 1L)
  9. .windowBy(Time.of(Duration.ofSeconds(1)))
  10. .output();
  11. Dataset<Pair<Object, Long>> output = Join.of(mapped, reduced)
  12. .by(e -> e, Pair::getFirst)
  13. .using((Object l, Pair<Object, Long> r, Collector<Long> c) -> {
  14. c.collect(r.getSecond());
  15. })
  16. .windowBy(Time.of(Duration.ofSeconds(1)))
  17. .output();
  18. ListDataSink<Pair<Object, Long>> sink = ListDataSink.get();
  19. output.persist(sink);
  20. reduced.persist(sink);
  21. FlowUnfolder.unfold(flow, Executor.getBasicOps());
  22. }

代码示例来源:origin: seznam/euphoria

  1. executor.submit(flow).get();

代码示例来源:origin: seznam/euphoria

  1. DAG<Operator<?, ?>> unfolded = FlowUnfolder.unfold(flow, Executor.getBasicOps());

代码示例来源:origin: seznam/euphoria

  1. private void run() {
  2. Flow flow = Flow.create();
  3. Dataset<Pair<ImmutableBytesWritable, Result>> ds = flow.createInput(
  4. Utils.getHBaseSource(input, conf.get()));
  5. FlatMap.of(ds)
  6. .using((Pair<ImmutableBytesWritable, Result> p, Collector<byte[]> c) -> {
  7. writeCellsAsBytes(p.getSecond(), c);
  8. })
  9. .output()
  10. .persist(Utils.getSink(output, conf.get()));
  11. LOG.info("Starting flow reading from {} and persisting to {}", input, output);
  12. executor.submit(flow).join();
  13. }

代码示例来源:origin: seznam/euphoria

  1. @Test
  2. public void test() {
  3. final Configuration conf = new Configuration();
  4. final String outputDir =
  5. Paths.get(tmp.getRoot().getAbsolutePath(), testName).toAbsolutePath().toString();
  6. final Flow flow = Flow.create();
  7. final S source = dataSinkTester.prepareDataSource();
  8. final T sink = dataSinkTester.buildSink(outputDir, conf, useLazyOutputFormat);
  9. MapElements.of(flow.createInput(source)).using(p -> p).output().persist(sink);
  10. final Executor executor = new LocalExecutor().setDefaultParallelism(4);
  11. executor.submit(flow).join();
  12. String[] files = new File(outputDir).list();
  13. assertNotNull(files);
  14. List<String> reduceOutputFileNames =
  15. Arrays.stream(files)
  16. .filter(file -> file.startsWith("part-r-"))
  17. .collect(Collectors.toList());
  18. assertEquals(expectedNumberOfReduceOutputs, reduceOutputFileNames.size());
  19. final List<O> output =
  20. reduceOutputFileNames
  21. .stream()
  22. .flatMap(dataSinkTester.extractOutputFunction(outputDir, conf))
  23. .collect(Collectors.toList());
  24. DatasetAssert.unorderedEquals(dataSinkTester.expectedOutput(), output);
  25. }

代码示例来源:origin: seznam/euphoria

  1. /**
  2. * Collects Avro record as JSON string
  3. *
  4. * @param outSink
  5. * @param inSource
  6. * @throws Exception
  7. */
  8. public static void runFlow(
  9. DataSink<String> outSink,
  10. DataSource<Pair<AvroKey<GenericData.Record>, NullWritable>> inSource)
  11. throws Exception {
  12. Flow flow = Flow.create("simple read avro");
  13. Dataset<Pair<AvroKey<GenericData.Record>, NullWritable>> input = flow.createInput(inSource);
  14. final Dataset<String> output =
  15. FlatMap.named("avro2csv").of(input).using(AvroSourceTest::apply).output();
  16. output.persist(outSink);
  17. Executor executor = new LocalExecutor();
  18. executor.submit(flow).get();
  19. }

代码示例来源:origin: seznam/euphoria

  1. .persist(Util.getStringSink(params));
  2. executorFactory.newExecutor(this.config, dataClasses()).submit(flow).get();

代码示例来源:origin: seznam/euphoria

  1. .build());
  2. executor.submit(flow).join();

代码示例来源:origin: seznam/euphoria

  1. .output().persist(out);
  2. executor.submit(flow).get();

相关文章