org.apache.hadoop.yarn.util.Records.newRecord()方法的使用及代码示例

x33g5p2x  于2022-01-28 转载在 其他  
字(11.8k)|赞(0)|评价(0)|浏览(96)

本文整理了Java中org.apache.hadoop.yarn.util.Records.newRecord方法的一些代码示例，展示了Records.newRecord的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Records.newRecord方法的具体详情如下：
包路径：org.apache.hadoop.yarn.util.Records
类名称：Records
方法名：newRecord

Records.newRecord介绍

暂无

代码示例

代码示例来源：origin: apache/flink

private static LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath) throws IOException {
  LocalResource localResource = Records.newRecord(LocalResource.class);
  FileStatus jarStat = fs.getFileStatus(remoteRsrcPath);
  localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
  localResource.setSize(jarStat.getLen());
  localResource.setTimestamp(jarStat.getModificationTime());
  localResource.setType(LocalResourceType.FILE);
  localResource.setVisibility(LocalResourceVisibility.APPLICATION);
  return localResource;
}

代码示例来源：origin: apache/incubator-gobblin

private void requestContainer(Optional<String> preferredNode) {
 Priority priority = Records.newRecord(Priority.class);
 priority.setPriority(0);
 Resource capability = Records.newRecord(Resource.class);
 int maxMemoryCapacity = this.maxResourceCapacity.get().getMemory();
 capability.setMemory(this.requestedContainerMemoryMbs <= maxMemoryCapacity ?
   this.requestedContainerMemoryMbs : maxMemoryCapacity);
 int maxCoreCapacity = this.maxResourceCapacity.get().getVirtualCores();
 capability.setVirtualCores(this.requestedContainerCores <= maxCoreCapacity ?
   this.requestedContainerCores : maxCoreCapacity);
 String[] preferredNodes = preferredNode.isPresent() ? new String[] {preferredNode.get()} : null;
 this.amrmClientAsync.addContainerRequest(
   new AMRMClient.ContainerRequest(capability, preferredNodes, null, priority));
}

代码示例来源：origin: apache/flink

ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

代码示例来源：origin: Qihoo360/XLearning

@Override
public CancelDelegationTokenResponse cancelDelegationToken(
  CancelDelegationTokenRequest request) throws IOException {
 if (!isAllowedDelegationTokenOp()) {
  throw new IOException(
    "Delegation Token can be cancelled only with kerberos authentication");
 }
 org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken();
 Token<MRDelegationTokenIdentifier> token =
   new Token<MRDelegationTokenIdentifier>(
     protoToken.getIdentifier().array(), protoToken.getPassword()
     .array(), new Text(protoToken.getKind()), new Text(
     protoToken.getService()));
 String user = UserGroupInformation.getCurrentUser().getUserName();
 jhsDTSecretManager.cancelToken(token, user);
 return Records.newRecord(CancelDelegationTokenResponse.class);
}

代码示例来源：origin: Qihoo360/XLearning

public static LocalResource createApplicationResource(FileSystem fs, Path path, LocalResourceType type)
  throws IOException {
 LocalResource localResource = Records.newRecord(LocalResource.class);
 FileStatus fileStatus = fs.getFileStatus(path);
 localResource.setResource(ConverterUtils.getYarnUrlFromPath(path));
 localResource.setSize(fileStatus.getLen());
 localResource.setTimestamp(fileStatus.getModificationTime());
 localResource.setType(type);
 localResource.setVisibility(LocalResourceVisibility.APPLICATION);
 return localResource;
}

代码示例来源：origin: apache/hive

private LocalResource createLocalResource(FileSystem remoteFs, Path file,
  LocalResourceType type, LocalResourceVisibility visibility) {
 FileStatus fstat = null;
 try {
  fstat = remoteFs.getFileStatus(file);
 } catch (IOException e) {
  e.printStackTrace();
 }
 URL resourceURL = ConverterUtils.getYarnUrlFromPath(file);
 long resourceSize = fstat.getLen();
 long resourceModificationTime = fstat.getModificationTime();
 LOG.info("Resource modification time: " + resourceModificationTime + " for " + file);
 LocalResource lr = Records.newRecord(LocalResource.class);
 lr.setResource(resourceURL);
 lr.setType(type);
 lr.setSize(resourceSize);
 lr.setVisibility(visibility);
 lr.setTimestamp(resourceModificationTime);
 return lr;
}

代码示例来源：origin: apache/drill

public Resource getCapability() {
 // Set up resource type requirements for ApplicationMaster
 Resource capability = Records.newRecord(Resource.class);
 capability.setMemory(memoryMb);
 capability.setVirtualCores(vCores);
 DoYUtil.callSetDiskIfExists(capability, disks);
 return capability;
}

代码示例来源：origin: Qihoo360/XLearning

Priority priority = Records.newRecord(Priority.class);
priority.setPriority(appPriority);
Resource workerCapability = Records.newRecord(Resource.class);
workerCapability.setMemory(workerMemory);
workerCapability.setVirtualCores(workerVCores);
 Resource psCapability = Records.newRecord(Resource.class);
 psCapability.setMemory(psMemory);
 psCapability.setVirtualCores(psVCores);

代码示例来源：origin: apache/ignite

/**
 * @param file Path.
 * @param fs File system.
 * @param type Local resource type.
 * @throws Exception If failed.
 */
public static LocalResource setupFile(Path file, FileSystem fs, LocalResourceType type)
  throws Exception {
  LocalResource resource = Records.newRecord(LocalResource.class);
  file = fs.makeQualified(file);
  FileStatus stat = fs.getFileStatus(file);
  resource.setResource(ConverterUtils.getYarnUrlFromPath(file));
  resource.setSize(stat.getLen());
  resource.setTimestamp(stat.getModificationTime());
  resource.setType(type);
  resource.setVisibility(LocalResourceVisibility.APPLICATION);
  return resource;
}

代码示例来源：origin: Qihoo360/XLearning

@Override
public RenewDelegationTokenResponse renewDelegationToken(
  RenewDelegationTokenRequest request) throws IOException {
 if (!isAllowedDelegationTokenOp()) {
  throw new IOException(
    "Delegation Token can be renewed only with kerberos authentication");
 }
 org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken();
 Token<MRDelegationTokenIdentifier> token =
   new Token<MRDelegationTokenIdentifier>(
     protoToken.getIdentifier().array(), protoToken.getPassword()
     .array(), new Text(protoToken.getKind()), new Text(
     protoToken.getService()));
 String user = UserGroupInformation.getCurrentUser().getShortUserName();
 long nextExpTime = jhsDTSecretManager.renewToken(token, user);
 RenewDelegationTokenResponse renewResponse = Records
   .newRecord(RenewDelegationTokenResponse.class);
 renewResponse.setNextExpirationTime(nextExpTime);
 return renewResponse;
}

代码示例来源：origin: apache/drill

assert memoryMb != 0;
Priority priorityRec = Records.newRecord(Priority.class);
priorityRec.setPriority(priority);
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(memoryMb);
capability.setVirtualCores(vCores);

代码示例来源：origin: apache/drill

private LocalResource createLocalResource(FileSystem remoteFs, Path file,
  LocalResourceType type, LocalResourceVisibility visibility) {
 FileStatus fstat = null;
 try {
  fstat = remoteFs.getFileStatus(file);
 } catch (IOException e) {
  e.printStackTrace();
 }
 URL resourceURL = ConverterUtils.getYarnUrlFromPath(file);
 long resourceSize = fstat.getLen();
 long resourceModificationTime = fstat.getModificationTime();
 LOG.info("Resource modification time: " + resourceModificationTime + " for " + file);
 LocalResource lr = Records.newRecord(LocalResource.class);
 lr.setResource(resourceURL);
 lr.setType(type);
 lr.setSize(resourceSize);
 lr.setVisibility(visibility);
 lr.setTimestamp(resourceModificationTime);
 return lr;
}

代码示例来源：origin: apache/incubator-gobblin

/**
 * Add a file as a Yarn {@link org.apache.hadoop.yarn.api.records.LocalResource}.
 *
 * @param fs a {@link FileSystem} instance
 * @param destFilePath the destination file path
 * @param resourceType the {@link org.apache.hadoop.yarn.api.records.LocalResourceType} of the file
 * @param resourceMap a {@link Map} of file names to their corresponding
 *                    {@link org.apache.hadoop.yarn.api.records.LocalResource}s
 * @throws IOException if there's something wrong adding the file as a
 *                     {@link org.apache.hadoop.yarn.api.records.LocalResource}
 */
public static void addFileAsLocalResource(FileSystem fs, Path destFilePath, LocalResourceType resourceType,
  Map<String, LocalResource> resourceMap) throws IOException {
 LocalResource fileResource = Records.newRecord(LocalResource.class);
 FileStatus fileStatus = fs.getFileStatus(destFilePath);
 fileResource.setResource(ConverterUtils.getYarnUrlFromPath(destFilePath));
 fileResource.setSize(fileStatus.getLen());
 fileResource.setTimestamp(fileStatus.getModificationTime());
 fileResource.setType(resourceType);
 fileResource.setVisibility(LocalResourceVisibility.APPLICATION);
 resourceMap.put(destFilePath.getName(), fileResource);
}

代码示例来源：origin: apache/flink

ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
ctx.setCommands(Collections.singletonList(launchCommand));
ctx.setLocalResources(taskManagerLocalResources);

代码示例来源：origin: apache/incubator-gobblin

private ContainerLaunchContext newContainerLaunchContext(Container container, String helixInstanceName)
  throws IOException {
 Path appWorkDir = GobblinClusterUtils.getAppWorkDirPath(this.fs, this.applicationName, this.applicationId);
 Path containerWorkDir = new Path(appWorkDir, GobblinYarnConfigurationKeys.CONTAINER_WORK_DIR_NAME);
 Map<String, LocalResource> resourceMap = Maps.newHashMap();
 addContainerLocalResources(new Path(appWorkDir, GobblinYarnConfigurationKeys.LIB_JARS_DIR_NAME), resourceMap);
 addContainerLocalResources(new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_JARS_DIR_NAME), resourceMap);
 addContainerLocalResources(
   new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_FILES_DIR_NAME), resourceMap);
 if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY)) {
  addRemoteAppFiles(this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY), resourceMap);
 }
 ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
 containerLaunchContext.setLocalResources(resourceMap);
 containerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
 containerLaunchContext.setCommands(Lists.newArrayList(buildContainerCommand(container, helixInstanceName)));
 if (UserGroupInformation.isSecurityEnabled()) {
  containerLaunchContext.setTokens(this.tokens.duplicate());
 }
 return containerLaunchContext;
}

代码示例来源：origin: apache/flink

/**
 * Creates a YARN resource for the remote object at the given location.
 *
 * @param remoteRsrcPath    remote location of the resource
 * @param resourceSize        size of the resource
 * @param resourceModificationTime last modification time of the resource
 *
 * @return YARN resource
 */
private static LocalResource registerLocalResource(
    Path remoteRsrcPath,
    long resourceSize,
    long resourceModificationTime) {
  LocalResource localResource = Records.newRecord(LocalResource.class);
  localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
  localResource.setSize(resourceSize);
  localResource.setTimestamp(resourceModificationTime);
  localResource.setType(LocalResourceType.FILE);
  localResource.setVisibility(LocalResourceVisibility.APPLICATION);
  return localResource;
}

代码示例来源：origin: apache/drill

/**
 * Given this generic description of an application, create the detailed YARN
 * application submission context required to launch the application.
 *
 * @param conf
 *          the YARN configuration obtained by reading the Hadoop
 *          configuration files
 * @return the completed application launch context for the given application
 * @throws IOException
 *           if localized resources are not found in the distributed file
 *           system (such as HDFS)
 */
public ContainerLaunchContext createLaunchContext(YarnConfiguration conf)
  throws IOException {
 // Set up the container launch context
 ContainerLaunchContext container = Records
   .newRecord(ContainerLaunchContext.class);
 // Set up the list of commands to run. Here, we assume that we run only
 // one command.
 container.setCommands(Collections.singletonList(getCommand()));
 // Add localized resources
 container.setLocalResources(resources);
 // Environment.
 container.setEnvironment(env);
 return container;
}

代码示例来源：origin: apache/ignite

Priority priority = Records.newRecord(Priority.class);
priority.setPriority(0);
      Resource capability = Records.newRecord(Resource.class);

代码示例来源：origin: apache/ignite

if (checkContainer(c)) {
  try {
    ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);

代码示例来源：origin: apache/incubator-gobblin

ContainerLaunchContext amContainerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
amContainerLaunchContext.setLocalResources(appMasterLocalResources);
amContainerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));

内容来源于网络，如有侵权，请联系作者删除！

records

相关文章

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多

    Records类方法

    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com