org.embulk.spi.Exec.newConfigDiff()方法的使用及代码示例

x33g5p2x  于2022-01-19 转载在 其他  
字(7.3k)|赞(0)|评价(0)|浏览(98)

本文整理了Java中org.embulk.spi.Exec.newConfigDiff()方法的一些代码示例,展示了Exec.newConfigDiff()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Exec.newConfigDiff()方法的具体详情如下:
包路径:org.embulk.spi.Exec
类名称:Exec
方法名:newConfigDiff

Exec.newConfigDiff介绍

暂无

代码示例

代码示例来源:origin: embulk/embulk

public void setOutputConfigDiff(ConfigDiff outputConfigDiff) {
  if (outputConfigDiff == null) {
    outputConfigDiff = Exec.newConfigDiff();
  }
  this.outputConfigDiff = outputConfigDiff;
}

代码示例来源:origin: embulk/embulk

@Override
  public ConfigDiff guess(ConfigSource config) {
    return Exec.newConfigDiff();
  }
}

代码示例来源:origin: embulk/embulk

public void setInputConfigDiff(ConfigDiff inputConfigDiff) {
  if (inputConfigDiff == null) {
    inputConfigDiff = Exec.newConfigDiff();
  }
  this.inputConfigDiff = inputConfigDiff;
}

代码示例来源:origin: embulk/embulk

@Override
  public ConfigDiff guess(ConfigSource config) {
    return Exec.newConfigDiff();
  }
}

代码示例来源:origin: embulk/embulk

@Override
public ConfigDiff resume(TaskSource taskSource,
    Schema schema, int taskCount,
    OutputPlugin.Control control) {
  control.run(taskSource);
  return Exec.newConfigDiff();
}

代码示例来源:origin: embulk/embulk

@Override
public ConfigDiff resume(TaskSource taskSource, Schema schema, int taskCount, Control control) {
  control.run(taskSource, schema, taskCount);
  return Exec.newConfigDiff();
}

代码示例来源:origin: embulk/embulk

@Override
public ConfigDiff resume(TaskSource taskSource,
    Schema schema, int taskCount,
    InputPlugin.Control control) {
  control.run(taskSource, schema, taskCount);
  return Exec.newConfigDiff();
}

代码示例来源:origin: embulk/embulk

public ConfigDiff resume(TaskSource taskSource,
    Schema schema, int taskCount,
    OutputPlugin.Control control) {
  control.run(taskSource);
  return Exec.newConfigDiff();
}

代码示例来源:origin: embulk/embulk

@Override
public ConfigDiff resume(TaskSource taskSource,
    int taskCount,
    FileOutputPlugin.Control control) {
  control.run(taskSource);
  return Exec.newConfigDiff();
}

代码示例来源:origin: embulk/embulk

public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) {
  control.run(Exec.newTaskSource(), 1);
  return Exec.newConfigDiff();
}

代码示例来源:origin: embulk/embulk

@Override
public ConfigDiff resume(final TaskSource taskSource, final int taskCount, final FileInputPlugin.Control control) {
  final PluginTask task = taskSource.loadTask(PluginTask.class);
  control.run(taskSource, taskCount);
  // build next config
  final ConfigDiff configDiff = Exec.newConfigDiff();
  // last_path
  if (task.getFiles().isEmpty()) {
    // keep the last value
    if (task.getLastPath().isPresent()) {
      configDiff.set("last_path", task.getLastPath().get());
    }
  } else {
    final List<String> files = new ArrayList<String>(task.getFiles());
    Collections.sort(files);
    configDiff.set("last_path", files.get(files.size() - 1));
  }
  return configDiff;
}

代码示例来源:origin: embulk/embulk

public ExecutionResult buildExecuteResultWithWarningException(Throwable ex) {
  ConfigDiff configDiff = Exec.newConfigDiff();
  if (inputConfigDiff != null) {
    configDiff.getNestedOrSetEmpty("in").merge(inputConfigDiff);
  }
  if (outputConfigDiff != null) {
    configDiff.getNestedOrSetEmpty("out").merge(outputConfigDiff);
  }
  ImmutableList.Builder<Throwable> ignoredExceptions = ImmutableList.builder();
  for (Throwable e : getExceptions()) {
    ignoredExceptions.add(e);
  }
  if (ex != null) {
    ignoredExceptions.add(ex);
  }
  return new ExecutionResult(configDiff, false, ignoredExceptions.build());
}

代码示例来源:origin: embulk/embulk

private ConfigDiff doGuess(ConfigSource config) {
  ConfigSource inputConfig = config.getNested("in");
  ConfigSource execConfig = config.getNestedOrGetEmpty("exec");
  InputPlugin input = newInputPlugin(inputConfig);
  ConfigDiff inputGuessed;
  if (input instanceof ConfigurableGuessInputPlugin) {
    inputGuessed = ((ConfigurableGuessInputPlugin) input).guess(execConfig, inputConfig);
  } else {
    try {
      inputGuessed = input.guess(inputConfig);
    } catch (AbstractMethodError ex) {
      // for backward compatibility with embulk v0.4 interface
      throw new UnsupportedOperationException(input.getClass().getSimpleName() + ".guess(ConfigSource) is not implemented. This input plugin does not support guessing.");
    }
  }
  ConfigDiff wrapped = Exec.newConfigDiff();
  wrapped.getNestedOrSetEmpty("in").merge(inputGuessed);
  return wrapped;
}

代码示例来源:origin: embulk/embulk

@Override
public void run(TaskSource taskSource, Schema schema, FileInput input, PageOutput pageOutput) {
  PluginTask task = taskSource.loadTask(PluginTask.class);
  final ConfigSource originalConfig = task.getOriginalConfig();
  final int guessParserSampleBufferBytes = task.getGuessParserSampleBufferBytes();
  // get sample buffer
  Buffer sample = readSample(input, guessParserSampleBufferBytes);
  // load guess plugins
  ImmutableList.Builder<GuessPlugin> builder = ImmutableList.builder();
  for (PluginType guessType : task.getGuessPluginTypes()) {
    GuessPlugin guess = Exec.newPlugin(GuessPlugin.class, guessType);
    builder.add(guess);
  }
  List<GuessPlugin> guesses = builder.build();
  // run guess plugins
  ConfigSource mergedConfig = originalConfig.deepCopy();
  ConfigDiff mergedGuessed = Exec.newConfigDiff();
  for (int i = 0; i < guesses.size(); i++) {
    ConfigDiff guessed = guesses.get(i).guess(originalConfig, sample);
    guessed = addAssumedDecoderConfigs(originalConfig, guessed);
    mergedGuessed.merge(guessed);
    mergedConfig.merge(mergedGuessed);
    if (!mergedConfig.equals(originalConfig)) {
      // config updated
      throw new GuessedNoticeError(mergedGuessed);
    }
  }
  throw new GuessedNoticeError(mergedGuessed);
}

代码示例来源:origin: embulk/embulk

@Override
public ConfigDiff transaction(ConfigSource config, int taskCount,
    FileOutputPlugin.Control control) {
  PluginTask task = config.loadConfig(PluginTask.class);
  control.run(task.dump());
  return Exec.newConfigDiff();
}

代码示例来源:origin: embulk/embulk

private ConfigDiff guessParserConfig(Buffer sample, ConfigSource config, List<PluginType> guessPlugins, final int guessParserSampleBufferBytes) {
  ConfigDiff lastGuessed = Exec.newConfigDiff();
  for (int i = 0; i < 10; i++) {

代码示例来源:origin: embulk/embulk-input-jdbc

public ConfigDiff guess(ConfigSource config)
{
  return Exec.newConfigDiff();
}

代码示例来源:origin: embulk/embulk-input-jdbc

protected ConfigDiff buildNextConfigDiff(PluginTask task, List<TaskReport> reports)
{
  ConfigDiff next = Exec.newConfigDiff();
  if (reports.size() > 0 && reports.get(0).has("last_record")) {
    next.set("last_record", reports.get(0).get(JsonNode.class, "last_record"));
  } else if (task.getLastRecord().isPresent()) {
    next.set("last_record", task.getLastRecord().get());
  }
  return next;
}

代码示例来源:origin: embulk/embulk-output-elasticsearch

@Override
  public ConfigDiff egestEmbulkData(final PluginTask task,
                   Schema schema,
                   int taskIndex,
                   List<TaskReport> taskReports)
  {
    long totalInserted = 0;
    for (TaskReport taskReport : taskReports) {
      if (taskReport.has("inserted")) {
        totalInserted += taskReport.get(Long.class, "inserted");
      }
    }

    log.info("Insert completed. {} records", totalInserted);
    // Re assign alias only when repale mode
    if (task.getMode().equals(Mode.REPLACE)) {
      client.reassignAlias(task.getAlias().orElse(null), task.getIndex(), task);
    }

    return Exec.newConfigDiff();
  }
}

代码示例来源:origin: embulk/embulk-input-s3

@Override
public ConfigDiff resume(TaskSource taskSource,
             int taskCount,
             FileInputPlugin.Control control)
{
  PluginTask task = taskSource.loadTask(getTaskClass());
  // validate task
  newS3Client(task);
  control.run(taskSource, taskCount);
  // build next config
  ConfigDiff configDiff = Exec.newConfigDiff();
  // last_path
  if (task.getIncremental()) {
    Optional<String> lastPath = task.getFiles().getLastPath(task.getLastPath());
    LOGGER.info("Incremental job, setting last_path to [{}]", lastPath.orElse(""));
    configDiff.set("last_path", lastPath);
  }
  return configDiff;
}

相关文章