com.alibaba.datax.common.util.Configuration.getNecessaryValue()方法的使用及代码示例

x33g5p2x  于2022-01-18 转载在 其他  
字(16.5k)|赞(0)|评价(0)|浏览(131)

本文整理了Java中com.alibaba.datax.common.util.Configuration.getNecessaryValue()方法的一些代码示例，展示了Configuration.getNecessaryValue()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Configuration.getNecessaryValue()方法的具体详情如下：
包路径：com.alibaba.datax.common.util.Configuration
类名称：Configuration
方法名：getNecessaryValue

Configuration.getNecessaryValue介绍

暂无

代码示例

代码示例来源：origin: ECNU-1X/DataX-Masking

public static String getEndpoint(Configuration conf) {
  return conf.getNecessaryValue("endpoint", ESWriterErrorCode.BAD_CONFIG_VALUE);
}

代码示例来源：origin: ECNU-1X/DataX-Masking

public static String getIndexName(Configuration conf) {
  return conf.getNecessaryValue("index", ESWriterErrorCode.BAD_CONFIG_VALUE);
}

代码示例来源：origin: ECNU-1X/DataX-Masking

public static void doPretreatment(Configuration originalConfig) {
  // 检查 username/password 配置（必填）
  originalConfig.getNecessaryValue(Key.USERNAME,
      DBUtilErrorCode.REQUIRED_VALUE);
  originalConfig.getNecessaryValue(Key.PASSWORD,
      DBUtilErrorCode.REQUIRED_VALUE);
  dealWhere(originalConfig);
  simplifyConf(originalConfig);
}

代码示例来源：origin: ECNU-1X/DataX-Masking

public static void checkNecessaryConfig(Configuration originalConfig) {
  originalConfig.getNecessaryValue(Key.ODPS_SERVER, OdpsReaderErrorCode.REQUIRED_VALUE);
  originalConfig.getNecessaryValue(Key.PROJECT, OdpsReaderErrorCode.REQUIRED_VALUE);
  originalConfig.getNecessaryValue(Key.TABLE, OdpsReaderErrorCode.REQUIRED_VALUE);
  if (null == originalConfig.getList(Key.COLUMN) ||
      originalConfig.getList(Key.COLUMN, String.class).isEmpty()) {
    throw DataXException.asDataXException(OdpsReaderErrorCode.REQUIRED_VALUE, "datax获取不到源表的列信息， 由于您未配置读取源头表的列信息. datax无法知道该抽取表的哪些字段的数据 " +
        "正确的配置方式是给 column 配置上您需要读取的列名称,用英文逗号分隔.");
  }
}

代码示例来源：origin: ECNU-1X/DataX-Masking

public static Configuration parseAccessIdAndKey(Configuration originalConfig) {
  String accessId = originalConfig.getString(Key.ACCESS_ID);
  String accessKey = originalConfig.getString(Key.ACCESS_KEY);
  // 只要 accessId,accessKey 二者配置了一个，就理解为是用户本意是要直接手动配置其 accessid/accessKey
  if (StringUtils.isNotBlank(accessId) || StringUtils.isNotBlank(accessKey)) {
    LOG.info("Try to get accessId/accessKey from your config.");
    //通过如下语句，进行检查是否确实配置了
    accessId = originalConfig.getNecessaryValue(Key.ACCESS_ID, OdpsReaderErrorCode.REQUIRED_VALUE);
    accessKey = originalConfig.getNecessaryValue(Key.ACCESS_KEY, OdpsReaderErrorCode.REQUIRED_VALUE);
    //检查完毕，返回即可
    return originalConfig;
  } else {
    Map<String, String> envProp = System.getenv();
    return getAccessIdAndKeyFromEnv(originalConfig, envProp);
  }
}

代码示例来源：origin: ECNU-1X/DataX-Masking

public static Configuration parseAccessIdAndKey(Configuration originalConfig) {
  String accessId = originalConfig.getString(Key.ACCESS_ID);
  String accessKey = originalConfig.getString(Key.ACCESS_KEY);
  // 只要 accessId,accessKey 二者配置了一个，就理解为是用户本意是要直接手动配置其 accessid/accessKey
  if (StringUtils.isNotBlank(accessId) || StringUtils.isNotBlank(accessKey)) {
    LOG.info("Try to get accessId/accessKey from your config.");
    //通过如下语句，进行检查是否确实配置了
    accessId = originalConfig.getNecessaryValue(Key.ACCESS_ID, OdpsWriterErrorCode.REQUIRED_VALUE);
    accessKey = originalConfig.getNecessaryValue(Key.ACCESS_KEY, OdpsWriterErrorCode.REQUIRED_VALUE);
    //检查完毕，返回即可
    return originalConfig;
  } else {
    Map<String, String> envProp = System.getenv();
    return getAccessIdAndKeyFromEnv(originalConfig, envProp);
  }
}

代码示例来源：origin: ECNU-1X/DataX-Masking

public static void checkNecessaryConfig(Configuration originalConfig) {
  originalConfig.getNecessaryValue(Key.ODPS_SERVER,
      OdpsWriterErrorCode.REQUIRED_VALUE);
  originalConfig.getNecessaryValue(Key.PROJECT,
      OdpsWriterErrorCode.REQUIRED_VALUE);
  originalConfig.getNecessaryValue(Key.TABLE,
      OdpsWriterErrorCode.REQUIRED_VALUE);
  if (null == originalConfig.getList(Key.COLUMN) ||
      originalConfig.getList(Key.COLUMN, String.class).isEmpty()) {
    throw DataXException.asDataXException(OdpsWriterErrorCode.REQUIRED_VALUE, "您未配置写入 ODPS 目的表的列信息. " +
        "正确的配置方式是给datax的 column 项配置上您需要读取的列名称,用英文逗号分隔 例如:  \"column\": [\"id\",\"name\"].");
  }
  // getBool 内部要求，值只能为 true,false 的字符串（大小写不敏感），其他一律报错，不再有默认配置
  Boolean truncate = originalConfig.getBool(Key.TRUNCATE);
  if (null == truncate) {
    throw DataXException.asDataXException(OdpsWriterErrorCode.REQUIRED_VALUE, "[truncate]是必填配置项, 意思是写入 ODPS 目的表前是否清空表/分区. " +
        "请您增加 truncate 的配置，根据业务需要选择上true 或者 false.");
  }
}

代码示例来源：origin: ECNU-1X/DataX-Masking

public static void doPretreatment(Configuration originalConfig, DataBaseType dataBaseType) {
  // 检查 username/password 配置（必填）
  originalConfig.getNecessaryValue(Key.USERNAME, DBUtilErrorCode.REQUIRED_VALUE);
  originalConfig.getNecessaryValue(Key.PASSWORD, DBUtilErrorCode.REQUIRED_VALUE);
  doCheckBatchSize(originalConfig);
  simplifyConf(originalConfig);
  dealColumnConf(originalConfig);
  dealWriteMode(originalConfig, dataBaseType);
}

代码示例来源：origin: ECNU-1X/DataX-Masking

public static void validateColumn(com.alibaba.datax.common.util.Configuration originalConfig){
  List<Configuration> columns = originalConfig.getListConfiguration(Key.COLUMN);
  if (columns == null || columns.isEmpty()) {
    throw DataXException.asDataXException(Hbase094xWriterErrorCode.REQUIRED_VALUE, "column为必填项，其形式为：column:[{\"index\": 0,\"name\": \"cf0:column0\",\"type\": \"string\"},{\"index\": 1,\"name\": \"cf1:column1\",\"type\": \"long\"}]");
  }
  for (Configuration aColumn : columns) {
    Integer index = aColumn.getInt(Key.INDEX);
    String type = aColumn.getNecessaryValue(Key.TYPE, Hbase094xWriterErrorCode.REQUIRED_VALUE);
    String name = aColumn.getNecessaryValue(Key.NAME, Hbase094xWriterErrorCode.REQUIRED_VALUE);
    ColumnType.getByTypeName(type);
    if(name.split(":").length != 2){
      throw DataXException.asDataXException(Hbase094xWriterErrorCode.ILLEGAL_VALUE, String.format("您column配置项中name配置的列格式[%s]不正确，name应该配置为 列族:列名  的形式, 如 {\"index\": 1,\"name\": \"cf1:q1\",\"type\": \"long\"}", name));
    }
    if(index == null || index < 0){
      throw DataXException.asDataXException(Hbase094xWriterErrorCode.ILLEGAL_VALUE, "您的column配置项不正确,配置项中中index为必填项,且为非负数，请检查并修改.");
    }
  }
}

代码示例来源：origin: ECNU-1X/DataX-Masking

private static void validateColumn(com.alibaba.datax.common.util.Configuration originalConfig){
  List<Configuration> columns = originalConfig.getListConfiguration(Key.COLUMN);
  if (columns == null || columns.isEmpty()) {
    throw DataXException.asDataXException(Hbase11xWriterErrorCode.REQUIRED_VALUE, "column为必填项，其形式为：column:[{\"index\": 0,\"name\": \"cf0:column0\",\"type\": \"string\"},{\"index\": 1,\"name\": \"cf1:column1\",\"type\": \"long\"}]");
  }
  for (Configuration aColumn : columns) {
    Integer index = aColumn.getInt(Key.INDEX);
    String type = aColumn.getNecessaryValue(Key.TYPE,Hbase11xWriterErrorCode.REQUIRED_VALUE);
    String name = aColumn.getNecessaryValue(Key.NAME,Hbase11xWriterErrorCode.REQUIRED_VALUE);
    ColumnType.getByTypeName(type);
    if(name.split(":").length != 2){
      throw DataXException.asDataXException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, String.format("您column配置项中name配置的列格式[%s]不正确，name应该配置为 列族:列名  的形式, 如 {\"index\": 1,\"name\": \"cf1:q1\",\"type\": \"long\"}", name));
    }
    if(index == null || index < 0){
      throw DataXException.asDataXException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, "您的column配置项不正确,配置项中中index为必填项,且为非负数，请检查并修改.");
    }
  }
}

代码示例来源：origin: ECNU-1X/DataX-Masking

private static void validateRowkeyColumn(com.alibaba.datax.common.util.Configuration originalConfig){
  List<Configuration> rowkeyColumn = originalConfig.getListConfiguration(Key.ROWKEY_COLUMN);
  if (rowkeyColumn == null || rowkeyColumn.isEmpty()) {
    throw DataXException.asDataXException(Hbase11xWriterErrorCode.REQUIRED_VALUE, "rowkeyColumn为必填项，其形式为：rowkeyColumn:[{\"index\": 0,\"type\": \"string\"},{\"index\": -1,\"type\": \"string\",\"value\": \"_\"}]");
  }
  int rowkeyColumnSize = rowkeyColumn.size();
  //包含{"index":0,"type":"string"} 或者 {"index":-1,"type":"string","value":"_"}
  for (Configuration aRowkeyColumn : rowkeyColumn) {
    Integer index = aRowkeyColumn.getInt(Key.INDEX);
    String type = aRowkeyColumn.getNecessaryValue(Key.TYPE,Hbase11xWriterErrorCode.REQUIRED_VALUE);
    ColumnType.getByTypeName(type);
    if(index == null ){
      throw DataXException.asDataXException(Hbase11xWriterErrorCode.REQUIRED_VALUE, "rowkeyColumn配置项中index为必填项");
    }
    //不能只有-1列,即rowkey连接串
    if(rowkeyColumnSize ==1 && index == -1){
      throw DataXException.asDataXException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, "rowkeyColumn配置项不能全为常量列,至少指定一个rowkey列");
    }
    if(index == -1){
      aRowkeyColumn.getNecessaryValue(Key.VALUE,Hbase11xWriterErrorCode.REQUIRED_VALUE);
    }
  }
}

代码示例来源：origin: ECNU-1X/DataX-Masking

public static void validateParameter(com.alibaba.datax.common.util.Configuration originalConfig) {
  originalConfig.getNecessaryValue(Key.HBASE_CONFIG, Hbase11xWriterErrorCode.REQUIRED_VALUE);
  originalConfig.getNecessaryValue(Key.TABLE, Hbase11xWriterErrorCode.REQUIRED_VALUE);
  Hbase11xHelper.validateMode(originalConfig);
  String encoding = originalConfig.getString(Key.ENCODING, Constant.DEFAULT_ENCODING);
  if (!Charset.isSupported(encoding)) {
    throw DataXException.asDataXException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, String.format("Hbasewriter 不支持您所配置的编码:[%s]", encoding));
  }
  originalConfig.set(Key.ENCODING, encoding);
  Boolean walFlag = originalConfig.getBool(Key.WAL_FLAG, false);
  originalConfig.set(Key.WAL_FLAG, walFlag);
  long writeBufferSize = originalConfig.getLong(Key.WRITE_BUFFER_SIZE,Constant.DEFAULT_WRITE_BUFFER_SIZE);
  originalConfig.set(Key.WRITE_BUFFER_SIZE, writeBufferSize);
}

代码示例来源：origin: ECNU-1X/DataX-Masking

public static void validateRowkeyColumn(com.alibaba.datax.common.util.Configuration originalConfig){
  List<Configuration> rowkeyColumn = originalConfig.getListConfiguration(Key.ROWKEY_COLUMN);
  if (rowkeyColumn == null || rowkeyColumn.isEmpty()) {
    throw DataXException.asDataXException(Hbase094xWriterErrorCode.REQUIRED_VALUE, "rowkeyColumn为必填项，其形式为：rowkeyColumn:[{\"index\": 0,\"type\": \"string\"},{\"index\": -1,\"type\": \"string\",\"value\": \"_\"}]");
  }
  int rowkeyColumnSize = rowkeyColumn.size();
  //包含{"index":0,"type":"string"} 或者 {"index":-1,"type":"string","value":"_"}
  for (Configuration aRowkeyColumn : rowkeyColumn) {
    Integer index = aRowkeyColumn.getInt(Key.INDEX);
    String type = aRowkeyColumn.getNecessaryValue(Key.TYPE, Hbase094xWriterErrorCode.REQUIRED_VALUE);
    ColumnType.getByTypeName(type);
    if(index == null ){
      throw DataXException.asDataXException(Hbase094xWriterErrorCode.REQUIRED_VALUE, "rowkeyColumn配置项中index为必填项");
    }
    //不能只有-1列,即rowkey连接串
    if(rowkeyColumnSize ==1 && index == -1){
      throw DataXException.asDataXException(Hbase094xWriterErrorCode.ILLEGAL_VALUE, "rowkeyColumn配置项不能全为常量列,至少指定一个rowkey列");
    }
    if(index == -1){
      aRowkeyColumn.getNecessaryValue(Key.VALUE, Hbase094xWriterErrorCode.REQUIRED_VALUE);
    }
  }
}

代码示例来源：origin: ECNU-1X/DataX-Masking

public  static  void validateMode(com.alibaba.datax.common.util.Configuration originalConfig){
  String mode = originalConfig.getNecessaryValue(Key.MODE, Hbase094xWriterErrorCode.REQUIRED_VALUE);
  ModeType modeType = ModeType.getByTypeName(mode);
  switch (modeType) {
    case Normal: {
      validateRowkeyColumn(originalConfig);
      validateColumn(originalConfig);
      validateVersionColumn(originalConfig);
      break;
    }
    default:
      throw DataXException.asDataXException(Hbase094xWriterErrorCode.ILLEGAL_VALUE,
          String.format("Hbase11xWriter不支持该 mode 类型:%s", mode));
  }
}

代码示例来源：origin: ECNU-1X/DataX-Masking

private  static  void validateMode(com.alibaba.datax.common.util.Configuration originalConfig){
  String mode = originalConfig.getNecessaryValue(Key.MODE,Hbase11xWriterErrorCode.REQUIRED_VALUE);
  ModeType modeType = ModeType.getByTypeName(mode);
  switch (modeType) {
    case Normal: {
      validateRowkeyColumn(originalConfig);
      validateColumn(originalConfig);
      validateVersionColumn(originalConfig);
      break;
    }
    default:
      throw DataXException.asDataXException(Hbase11xWriterErrorCode.ILLEGAL_VALUE,
          String.format("Hbase11xWriter不支持该 mode 类型:%s", mode));
  }
}

代码示例来源：origin: ECNU-1X/DataX-Masking

public static void validateParameter(com.alibaba.datax.common.util.Configuration originalConfig) {
  originalConfig.getNecessaryValue(Key.HBASE_CONFIG, Hbase094xWriterErrorCode.REQUIRED_VALUE);
  originalConfig.getNecessaryValue(Key.TABLE, Hbase094xWriterErrorCode.REQUIRED_VALUE);
  Hbase094xHelper.validateMode(originalConfig);
  String encoding = originalConfig.getString(Key.ENCODING, Constant.DEFAULT_ENCODING);
  if (!Charset.isSupported(encoding)) {
    throw DataXException.asDataXException(Hbase094xWriterErrorCode.ILLEGAL_VALUE, String.format("Hbasewriter 不支持您所配置的编码:[%s]", encoding));
  }
  originalConfig.set(Key.ENCODING, encoding);
  Boolean autoFlush = originalConfig.getBool(Key.AUTO_FLUSH, false);
  //本期设置autoflush 一定为flase,通过hbase writeBufferSize来控制每次flush大小
  originalConfig.set(Key.AUTO_FLUSH,false);
  Boolean walFlag = originalConfig.getBool(Key.WAL_FLAG, false);
  originalConfig.set(Key.WAL_FLAG, walFlag);
  long writeBufferSize = originalConfig.getLong(Key.WRITE_BUFFER_SIZE,Constant.DEFAULT_WRITE_BUFFER_SIZE);
  originalConfig.set(Key.WRITE_BUFFER_SIZE, writeBufferSize);
}

代码示例来源：origin: ECNU-1X/DataX-Masking

@Override
public void init() {
  this.taskConfig = super.getPluginJobConf();
  this.sourceFiles = this.taskConfig.getList(Constant.SOURCE_FILES, String.class);
  this.specifiedFileType = this.taskConfig.getNecessaryValue(Key.FILETYPE, HdfsReaderErrorCode.REQUIRED_VALUE);
  this.encoding = this.taskConfig.getString(com.alibaba.datax.plugin.unstructuredstorage.reader.Key.ENCODING, "UTF-8");
  this.dfsUtil = new DFSUtil(this.taskConfig);
  this.bufferSize = this.taskConfig.getInt(com.alibaba.datax.plugin.unstructuredstorage.reader.Key.BUFFER_SIZE,
      com.alibaba.datax.plugin.unstructuredstorage.reader.Constant.DEFAULT_BUFFER_SIZE);
}

代码示例来源：origin: ECNU-1X/DataX-Masking

private static void validateVersionColumn(com.alibaba.datax.common.util.Configuration originalConfig){
    Configuration versionColumn = originalConfig.getConfiguration(Key.VERSION_COLUMN);
    //为null,表示用当前时间;指定列,需要index
    if(versionColumn != null){
      Integer index = versionColumn.getInt(Key.INDEX);
      if(index == null ){
        throw DataXException.asDataXException(Hbase11xWriterErrorCode.REQUIRED_VALUE, "versionColumn配置项中index为必填项");
      }
      if(index == -1){
        //指定时间,需要index=-1,value
        versionColumn.getNecessaryValue(Key.VALUE,Hbase11xWriterErrorCode.REQUIRED_VALUE);
      }else if(index < 0){
        throw DataXException.asDataXException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, "您versionColumn配置项中index配置不正确,只能取-1或者非负数");
      }
    }
  }
}

代码示例来源：origin: ECNU-1X/DataX-Masking

public static void validateVersionColumn(com.alibaba.datax.common.util.Configuration originalConfig){
    Configuration versionColumn = originalConfig.getConfiguration(Key.VERSION_COLUMN);
    //为null,表示用当前时间;指定列,需要index
    if(versionColumn != null){
      Integer index = versionColumn.getInt(Key.INDEX);
      if(index == null ){
        throw DataXException.asDataXException(Hbase094xWriterErrorCode.REQUIRED_VALUE, "versionColumn配置项中index为必填项");
      }
      if(index == -1){
        //指定时间,需要index=-1,value
        versionColumn.getNecessaryValue(Key.VALUE, Hbase094xWriterErrorCode.REQUIRED_VALUE);
      }else if(index < 0){
        throw DataXException.asDataXException(Hbase094xWriterErrorCode.ILLEGAL_VALUE, "您versionColumn配置项中index配置不正确,只能取-1或者非负数");
      }
    }
  }
}

代码示例来源：origin: ECNU-1X/DataX-Masking

private static String validateMode(Configuration  originalConfig) {
  String mode = originalConfig.getNecessaryValue(Key.MODE,Hbase094xReaderErrorCode.REQUIRED_VALUE);
  List<Map> column = originalConfig.getList(Key.COLUMN, Map.class);
  if (column == null || column.isEmpty()) {
    throw DataXException.asDataXException(Hbase094xReaderErrorCode.REQUIRED_VALUE, "您配置的column为空,Hbase必须配置 column，其形式为：column:[{\"name\": \"cf0:column0\",\"type\": \"string\"},{\"name\": \"cf1:column1\",\"type\": \"long\"}]");
  }
  ModeType modeType = ModeType.getByTypeName(mode);
  switch (modeType) {
    case Normal: {
      // normal 模式不需要配置 maxVersion，需要配置 column，并且 column 格式为 Map 风格
      String maxVersion = originalConfig.getString(Key.MAX_VERSION);
      Validate.isTrue(maxVersion == null, "您配置的是 normal 模式读取 hbase 中的数据，所以不能配置无关项：maxVersion");
      // 通过 parse 进行 column 格式的进一步检查
      Hbase094xHelper.parseColumnOfNormalMode(column);
      break;
    }
    case MultiVersionFixedColumn:{
      // multiVersionFixedColumn 模式需要配置 maxVersion
      checkMaxVersion(originalConfig, mode);
      Hbase094xHelper.parseColumnOfMultiversionMode(column);
      break;
    }
    default:
      throw DataXException.asDataXException(Hbase094xReaderErrorCode.ILLEGAL_VALUE,
          String.format("Hbase11xReader不支持该 mode 类型:%s", mode));
  }
  return mode;
}

内容来源于网络，如有侵权，请联系作者删除！

Configuration

相关文章

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com