org.apache.hadoop.yarn.ipc.YarnRPC类的使用及代码示例

x33g5p2x  于2022-02-05 转载在 其他  
字(10.8k)|赞(0)|评价(0)|浏览(119)

本文整理了Java中org.apache.hadoop.yarn.ipc.YarnRPC类的一些代码示例，展示了YarnRPC类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。YarnRPC类的具体详情如下：
包路径：org.apache.hadoop.yarn.ipc.YarnRPC
类名称：YarnRPC

YarnRPC介绍

[英]Abstraction to get the RPC implementation for Yarn.
[中]获取纱线的RPC实现的抽象。

代码示例

代码示例来源：origin: apache/incubator-gobblin

private static void getJhToken(Configuration conf, Credentials cred) throws IOException {
 YarnRPC rpc = YarnRPC.create(conf);
 final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS);
 LOG.debug("Connecting to HistoryServer at: " + serviceAddr);
 HSClientProtocol hsProxy =
   (HSClientProtocol) rpc.getProxy(HSClientProtocol.class, NetUtils.createSocketAddr(serviceAddr), conf);
 LOG.info("Pre-fetching JH token from job history server");
 Token<?> jhToken = null;
 try {
  jhToken = getDelegationTokenFromHS(hsProxy, conf);
 } catch (Exception exc) {
  throw new IOException("Failed to fetch JH token.", exc);
 }
 if (jhToken == null) {
  LOG.error("getDelegationTokenFromHS() returned null");
  throw new IOException("Unable to fetch JH token.");
 }
 LOG.info("Created JH token: " + jhToken.toString());
 LOG.info("Token kind: " + jhToken.getKind());
 LOG.info("Token id: " + Arrays.toString(jhToken.getIdentifier()));
 LOG.info("Token service: " + jhToken.getService());
 cred.addToken(jhToken.getService(), jhToken);
}

代码示例来源：origin: Qihoo360/XLearning

protected void serviceStart() throws Exception {
 Configuration conf = new XLearningConfiguration();
 YarnRPC rpc = YarnRPC.create(conf);
 initializeWebApp(conf);
 InetSocketAddress address = conf.getSocketAddr(
   XLearningConfiguration.XLEARNING_HISTORY_BIND_HOST,
   XLearningConfiguration.XLEARNING_HISTORY_ADDRESS,
   conf.get(XLearningConfiguration.XLEARNING_HISTORY_ADDRESS, XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_ADDRESS),
   conf.getInt(XLearningConfiguration.XLEARNING_HISTORY_PORT, XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_PORT));
 server =
   rpc.getServer(HSClientProtocol.class, protocolHandler, address,
     conf, jhsDTSecretManager,
     conf.getInt(XLearningConfiguration.XLEARNING_HISTORY_CLIENT_THREAD_COUNT,
       XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_CLIENT_THREAD_COUNT));
 // Enable service authorization?
 if (conf.getBoolean(
   CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
   false)) {
  server.refreshServiceAcl(conf, new ClientHSPolicyProvider());
 }
 server.start();
 this.bindAddress = conf.updateConnectAddr(XLearningConfiguration.XLEARNING_HISTORY_BIND_HOST,
   XLearningConfiguration.XLEARNING_HISTORY_ADDRESS,
   conf.get(XLearningConfiguration.XLEARNING_HISTORY_ADDRESS, XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_ADDRESS),
   server.getListenerAddress());
 LOG.info("Instantiated HistoryClientService at " + this.bindAddress);
 super.serviceStart();
}

代码示例来源：origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager

@Test
public void testRMStartWithDecommissionedNode() throws Exception {
 String excludeFile = "excludeFile";
 createExcludeFile(excludeFile);
 YarnConfiguration conf = new YarnConfiguration();
 conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
   excludeFile);
 MockRM rm = new MockRM(conf) {
  protected ClientRMService createClientRMService() {
   return new ClientRMService(this.rmContext, scheduler,
     this.rmAppManager, this.applicationACLsManager, this.queueACLsManager,
     this.getRMContext().getRMDelegationTokenSecretManager());
  };
 };
 rm.start();
 YarnRPC rpc = YarnRPC.create(conf);
 InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
 LOG.info("Connecting to ResourceManager at " + rmAddress);
 ApplicationClientProtocol client =
   (ApplicationClientProtocol) rpc
     .getProxy(ApplicationClientProtocol.class, rmAddress, conf);
 // Make call
 GetClusterNodesRequest request =
   GetClusterNodesRequest.newInstance(EnumSet.allOf(NodeState.class));
 List<NodeReport> nodeReports = client.getClusterNodes(request).getNodeReports();
 Assert.assertEquals(1, nodeReports.size());
 rm.stop();
 rpc.stopProxy(client, conf);
 new File(excludeFile).delete();
}

代码示例来源：origin: ch.cern.hadoop/hadoop-yarn-common

@Test
public void testUnknownCall() {
 Configuration conf = new Configuration();
 conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class
   .getName());
 YarnRPC rpc = YarnRPC.create(conf);
 String bindAddr = "localhost:0";
 InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
 Server server = rpc.getServer(ContainerManagementProtocol.class,
   new DummyContainerManager(), addr, conf, null, 1);
 server.start();
 // Any unrelated protocol would do
 ApplicationClientProtocol proxy = (ApplicationClientProtocol) rpc.getProxy(
   ApplicationClientProtocol.class, NetUtils.getConnectAddress(server), conf);
 try {
  proxy.getNewApplication(Records
    .newRecord(GetNewApplicationRequest.class));
  Assert.fail("Excepted RPC call to fail with unknown method.");
 } catch (YarnException e) {
  Assert.assertTrue(e.getMessage().matches(
    "Unknown method getNewApplication called on.*"
      + "org.apache.hadoop.yarn.proto.ApplicationClientProtocol"
      + "\\$ApplicationClientProtocolService\\$BlockingInterface protocol."));
 } catch (Exception e) {
  e.printStackTrace();
 }
}

代码示例来源：origin: org.apache.hadoop/hadoop-yarn-common

@Override
 public T run() {
  return (T) rpc.getProxy(protocol, serverAddress, conf);
 }
});

代码示例来源：origin: org.apache.apex/apex-engine

public YarnClientHelper(Configuration conf)
{
 // Set up the configuration and RPC
 this.conf = conf;
 this.rpc = YarnRPC.create(conf);
}

代码示例来源：origin: org.apache.hadoop/hadoop-yarn-common

public Server getServer(Class protocol, Object instance,
  InetSocketAddress addr, Configuration conf,
  SecretManager<? extends TokenIdentifier> secretManager,
  int numHandlers) {
 return getServer(protocol, instance, addr, conf, secretManager, numHandlers,
   null);
}

代码示例来源：origin: ch.cern.hadoop/hadoop-yarn-client

private boolean tryCloseProxy(
  ContainerManagementProtocolProxyData proxy) {
 proxy.activeCallers--;
 if (proxy.scheduledForClose && proxy.activeCallers < 0) {
  LOG.info("Closing proxy : " + proxy.containerManagerBindAddr);
  cmProxy.remove(proxy.containerManagerBindAddr);
  try {
   rpc.stopProxy(proxy.getContainerManagementProtocol(), conf);
  } finally {
   this.notifyAll();
  }
  return true;
 }
 return false;
}

代码示例来源：origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager

rm.start();
YarnRPC rpc = YarnRPC.create(conf);
InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
LOG.info("Connecting to ResourceManager at " + rmAddress);
ApplicationClientProtocol client =
  (ApplicationClientProtocol) rpc
    .getProxy(ApplicationClientProtocol.class, rmAddress, conf);
rpc.stopProxy(client, conf);
new File(excludeFile).delete();

代码示例来源：origin: io.hops/hadoop-yarn-common

@Test
public void testUnknownCall() {
 Configuration conf = new Configuration();
 conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class
   .getName());
 YarnRPC rpc = YarnRPC.create(conf);
 String bindAddr = "localhost:0";
 InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
 Server server = rpc.getServer(ContainerManagementProtocol.class,
   new DummyContainerManager(), addr, conf, null, 1);
 server.start();
 // Any unrelated protocol would do
 ApplicationClientProtocol proxy = (ApplicationClientProtocol) rpc.getProxy(
   ApplicationClientProtocol.class, NetUtils.getConnectAddress(server), conf);
 try {
  proxy.getNewApplication(Records
    .newRecord(GetNewApplicationRequest.class));
  Assert.fail("Excepted RPC call to fail with unknown method.");
 } catch (YarnException e) {
  Assert.assertTrue(e.getMessage().matches(
    "Unknown method getNewApplication called on.*"
      + "org.apache.hadoop.yarn.proto.ApplicationClientProtocol"
      + "\\$ApplicationClientProtocolService\\$BlockingInterface protocol."));
 } catch (Exception e) {
  e.printStackTrace();
 }
}

代码示例来源：origin: org.apache.hadoop/hadoop-mapreduce-client-common

@Override
 public MRClientProtocol run() {
  return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class,
    hsAddress, conf);
 }
});

代码示例来源：origin: alibaba/mpich2-yarn

/**
 * Constructor, connect to Resource Manager
 *
 * @throws IOException
 */
public ApplicationMaster() throws IOException {
 super(ApplicationMaster.class.getName());
 // Set up the configuration and RPC
 conf = new MPIConfiguration();
 rpc = YarnRPC.create(conf);
 dfs = FileSystem.get(conf);
}

代码示例来源：origin: com.github.jiayuhan-it/hadoop-yarn-common

public Server getServer(Class protocol, Object instance,
  InetSocketAddress addr, Configuration conf,
  SecretManager<? extends TokenIdentifier> secretManager,
  int numHandlers) {
 return getServer(protocol, instance, addr, conf, secretManager, numHandlers,
   null);
}

代码示例来源：origin: org.apache.hadoop/hadoop-yarn-client

private boolean tryCloseProxy(
  ContainerManagementProtocolProxyData proxy) {
 proxy.activeCallers--;
 if (proxy.scheduledForClose && proxy.activeCallers < 0) {
  if (LOG.isDebugEnabled()) {
   LOG.debug("Closing proxy : " + proxy.containerManagerBindAddr);
  }
  cmProxy.remove(proxy.containerManagerBindAddr);
  try {
   rpc.stopProxy(proxy.getContainerManagementProtocol(), conf);
  } finally {
   this.notifyAll();
  }
  return true;
 }
 return false;
}

代码示例来源：origin: org.apache.hadoop/hadoop-yarn-common

@Override
 public T run() {
  return (T) YarnRPC.create(conf).getProxy(protocol, rmAddress, conf);
 }
});

代码示例来源：origin: org.apache.hadoop/hadoop-yarn-server-nodemanager

Server createServer() {
 Configuration conf = getConfig();
 YarnRPC rpc = YarnRPC.create(conf);
 if (UserGroupInformation.isSecurityEnabled()) {
  secretManager = new LocalizerTokenSecretManager();      
 }
 
 Server server = rpc.getServer(LocalizationProtocol.class, this,
   localizationServerAddress, conf, secretManager, 
   conf.getInt(YarnConfiguration.NM_LOCALIZER_CLIENT_THREAD_COUNT, 
     YarnConfiguration.DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT));
 
 // Enable service authorization?
 if (conf.getBoolean(
   CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
   false)) {
  server.refreshServiceAcl(conf, NMPolicyProvider.getInstance());
 }
 
 return server;
}

代码示例来源：origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager

YarnRPC rpc = YarnRPC.create(conf);
InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
LOG.info("Connecting to ResourceManager at " + rmAddress);
ApplicationClientProtocol client =
  (ApplicationClientProtocol) rpc.getProxy(
    ApplicationClientProtocol.class, rmAddress, conf);
Assert.assertTrue(nodeToLabels.get(node2).containsAll(Arrays.asList("y")));
rpc.stopProxy(client, conf);
rm.close();

代码示例来源：origin: org.apache.hadoop/hadoop-yarn-server-common

conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class
  .getName());
YarnRPC rpc = YarnRPC.create(conf);
String bindAddr = "localhost:0";
InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
Server server = rpc.getServer(ContainerManagementProtocol.class,
  new DummyContainerManager(), addr, conf, null, 1);
server.start();
ApplicationClientProtocol proxy = (ApplicationClientProtocol) rpc.getProxy(
  ApplicationClientProtocol.class, NetUtils.getConnectAddress(server),
  conf);

代码示例来源：origin: ch.cern.hadoop/hadoop-yarn-common

@Override
 public T run() {
  return (T) rpc.getProxy(protocol, serverAddress, conf);
 }
});

代码示例来源：origin: org.apache.twill/twill-yarn

public Hadoop20YarnAMClient(Configuration conf) {
 super(ApplicationConstants.AM_CONTAINER_ID_ENV);
 this.amrmClient = new AMRMClientImpl(containerId.getApplicationAttemptId());
 this.amrmClient.init(conf);
 this.nmClient = new Hadoop20YarnNMClient(YarnRPC.create(conf), conf);
}

内容来源于网络，如有侵权，请联系作者删除！

YarnRPC

相关文章

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com