如何从java代码运行hadoop hdfs命令

pw9qyyiw  于 2021-05-31  发布在  Hadoop
关注(0)|答案(5)|浏览(385)

我是hadoop新手!如何从java代码运行一些hdfs命令?我一直在用java代码和hdfs命令直接从clouderavm的终端成功地测试mapreduce,但是现在我想学习如何用java代码来做。我一直在寻找任何可以学习的材料,但还没有找到。谢谢

vql8enpb

vql8enpb1#

您可以在java代码中使用文件系统api来执行hdfs命令。https://hadoop.apache.org/docs/r2.8.2/api/org/apache/hadoop/fs/filesystem.html
请查找以下示例代码。

  1. package com.hadoop.FilesystemClasses;
  2. import java.io.IOException;
  3. import org.apache.hadoop.fs.FileSystem;
  4. import org.apache.hadoop.conf.Configuration;
  5. import org.apache.hadoop.fs.Path;
  6. import org.apache.log4j.Logger;
  7. import com.hadoop.Constants.Constants;
  8. public class HdfsFileSystemTasks {
  9. public static Logger logger = Logger.getLogger(HdfsFileSystemTasks.class
  10. .getName());
  11. public FileSystem configureFilesystem(String coreSitePath,
  12. String hdfsSitePath) {
  13. FileSystem fileSystem = null;
  14. try {
  15. Configuration conf = new Configuration();
  16. Path hdfsCoreSitePath = new Path(coreSitePath);
  17. Path hdfsHDFSSitePath = new Path(hdfsSitePath);
  18. conf.addResource(hdfsCoreSitePath);
  19. conf.addResource(hdfsHDFSSitePath);
  20. fileSystem = FileSystem.get(conf);
  21. return fileSystem;
  22. } catch (Exception ex) {
  23. ex.printStackTrace();
  24. return fileSystem;
  25. }
  26. }
  27. public String writeToHDFS(FileSystem fileSystem, String sourcePath,
  28. String destinationPath) {
  29. try {
  30. Path inputPath = new Path(sourcePath);
  31. Path outputPath = new Path(destinationPath);
  32. fileSystem.copyFromLocalFile(inputPath, outputPath);
  33. return Constants.SUCCESS;
  34. } catch (IOException ex) {
  35. ex.printStackTrace();
  36. return Constants.FAILURE;
  37. }
  38. }
  39. public String readFileFromHdfs(FileSystem fileSystem, String hdfsStorePath,
  40. String localSystemPath) {
  41. try {
  42. Path hdfsPath = new Path(hdfsStorePath);
  43. Path localPath = new Path(localSystemPath);
  44. fileSystem.copyToLocalFile(hdfsPath, localPath);
  45. return Constants.SUCCESS;
  46. } catch (IOException ex) {
  47. ex.printStackTrace();
  48. return Constants.FAILURE;
  49. }
  50. }
  51. public String deleteHdfsDirectory(FileSystem fileSystem,
  52. String hdfsStorePath) {
  53. try {
  54. Path hdfsPath = new Path(hdfsStorePath);
  55. if (fileSystem.exists(hdfsPath)) {
  56. fileSystem.delete(hdfsPath);
  57. logger.info("Directory{} Deleted Successfully "
  58. + hdfsPath);
  59. } else {
  60. logger.info("Input Directory{} does not Exists " + hdfsPath);
  61. }
  62. return Constants.SUCCESS;
  63. } catch (Exception ex) {
  64. System.out
  65. .println("Some exception occurred while reading file from hdfs");
  66. ex.printStackTrace();
  67. return Constants.FAILURE;
  68. }
  69. }
  70. public String deleteLocalDirectory(FileSystem fileSystem,
  71. String localStorePath) {
  72. try {
  73. Path localPath = new Path(localStorePath);
  74. if (fileSystem.exists(localPath)) {
  75. fileSystem.delete(localPath);
  76. logger.info("Input Directory{} Deleted Successfully "
  77. + localPath);
  78. } else {
  79. logger.info("Input Directory{} does not Exists " + localPath);
  80. }
  81. return Constants.SUCCESS;
  82. } catch (Exception ex) {
  83. System.out
  84. .println("Some exception occurred while reading file from hdfs");
  85. ex.printStackTrace();
  86. return Constants.FAILURE;
  87. }
  88. }
  89. public void closeFileSystem(FileSystem fileSystem) {
  90. try {
  91. fileSystem.close();
  92. } catch (Exception ex) {
  93. ex.printStackTrace();
  94. System.out.println("Unable to close Hadoop filesystem : " + ex);
  95. }
  96. }
  97. }
  98. package com.hadoop.FileSystemTasks;
  99. import com.hadoop.Constants.HDFSParameters;
  100. import com.hadoop.Constants.HdfsFilesConstants;
  101. import com.hadoop.Constants.LocalFilesConstants;
  102. import com.hadoop.FilesystemClasses.HdfsFileSystemTasks;
  103. import org.apache.hadoop.fs.FileSystem;
  104. import org.apache.log4j.Logger;
  105. public class ExecuteFileSystemTasks {
  106. public static Logger logger = Logger.getLogger(ExecuteFileSystemTasks.class
  107. .getName());
  108. public static void main(String[] args) {
  109. HdfsFileSystemTasks hdfsFileSystemTasks = new HdfsFileSystemTasks();
  110. FileSystem fileSystem = hdfsFileSystemTasks.configureFilesystem(
  111. HDFSParameters.CORE_SITE_XML_PATH,
  112. HDFSParameters.HDFS_SITE_XML_PATH);
  113. logger.info("File System Object {} " + fileSystem);
  114. String fileWriteStatus = hdfsFileSystemTasks.writeToHDFS(fileSystem,
  115. LocalFilesConstants.SALES_DATA_LOCAL_PATH,
  116. HdfsFilesConstants.HDFS_SOURCE_DATA_PATH);
  117. logger.info("File Write Status{} " + fileWriteStatus);
  118. String filereadStatus = hdfsFileSystemTasks.readFileFromHdfs(
  119. fileSystem, HdfsFilesConstants.HDFS_DESTINATION_DATA_PATH
  120. + "/MR_Job_Res2/part-r-00000",
  121. LocalFilesConstants.MR_RESULTS_LOCALL_PATH);
  122. logger.info("File Read Status{} " + filereadStatus);
  123. String deleteDirStatus = hdfsFileSystemTasks.deleteHdfsDirectory(
  124. fileSystem, HdfsFilesConstants.HDFS_DESTINATION_DATA_PATH
  125. + "/MR_Job_Res2");
  126. hdfsFileSystemTasks.closeFileSystem(fileSystem);
  127. }
  128. }
展开查看全部
dxpyg8gm

dxpyg8gm2#

我想这可能对你有帮助
我很好地使用它执行shell命令

  1. public class JavaRunShell {
  2. public static void main(String[] args){
  3. try {
  4. String shpath=" your command";
  5. Process ps = Runtime.getRuntime().exec(shpath);
  6. ps.waitFor();
  7. }
  8. catch (Exception e) {
  9. e.printStackTrace();
  10. }
  11. }
  12. }
wn9m85ua

wn9m85ua3#

您可以在java代码中使用文件系统api与hdfs交互。

shyt4zoc

shyt4zoc4#

@我试过运行你的代码,但总是出错。这就是我犯的错误

  1. java.io.IOException: Cannot run program "your": CreateProcess error=2, The system cannot
  2. find the file specified
  3. at java.lang.ProcessBuilder.start(Unknown Source)
  4. at java.lang.Runtime.exec(Unknown Source)
  5. at java.lang.Runtime.exec(Unknown Source)
  6. at java.lang.Runtime.exec(Unknowenter code heren Source)
  7. at jrs.main(jrs.java:5)
6jjcrrmo

6jjcrrmo5#

正如jagrut所提到的,可以在java代码中使用文件系统api与hdfs命令交互。下面是我试图检查hdfs中是否存在特定目录的示例代码。如果存在,则删除该hdfs目录。

  1. Configuration conf = new Configuration();
  2. Job job = new Job(conf,"HDFS Connect");
  3. FileSystem fs = FileSystem.get(conf);
  4. Path outputPath = new Path("/user/cloudera/hdfsPath");
  5. if(fs.exists(outputPath))
  6. fs.delete(outputPath);

你也可以参考给定的博客以供进一步参考-
https://dzone.com/articles/working-with-the-hadoop-file-system-api, https://hadoop.apache.org/docs/r2.8.2/api/org/apache/hadoop/fs/filesystem.htmlhttpshttp://blog.knoldus.com/2017/04/16/working-with-hadoop-filesystem-api/

相关问题