本文整理了Java中org.apache.hadoop.hive.ql.metadata.Hive.copyFiles()
方法的一些代码示例,展示了Hive.copyFiles()
的具体用法。这些代码示例主要来源于Github
/Stackoverflow
/Maven
等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Hive.copyFiles()
方法的具体详情如下:
包路径:org.apache.hadoop.hive.ql.metadata.Hive
类名称:Hive
方法名:copyFiles
[英]Copy files. This handles building the mapping for buckets and such between the source and destination
[中]复制文件。这将处理为源和目标之间的bucket等构建映射
代码示例来源:origin: apache/hive
copyFiles(conf, fs, srcs, srcFs, destf, isSrcLocal, isOverwrite,
newFiles, isFullAcidTable && !isBucketed, isManaged);
代码示例来源:origin: apache/drill
moveAcidFiles(srcFs, srcs, destf, newFiles);
} else {
copyFiles(conf, fs, srcs, srcFs, destf, isSrcLocal, newFiles);
代码示例来源:origin: apache/hive
@Test
public void testCopyNewFilesOnDifferentFileSystem() throws IOException {
Path sourcePath = new Path(sourceFolder.getRoot().getAbsolutePath());
sourceFolder.newFile("000000_0");
sourceFolder.newFile("000001_0");
sourceFolder.newFile("000000_0.gz");
sourceFolder.newFile("000001_0.gz");
Path targetPath = new Path(targetFolder.getRoot().getAbsolutePath());
// Simulate different filesystems by returning a different URI
FileSystem spyTargetFs = Mockito.spy(targetPath.getFileSystem(hiveConf));
Mockito.when(spyTargetFs.getUri()).thenReturn(URI.create("hdfs://" + targetPath.toUri().getPath()));
try {
Hive.copyFiles(hiveConf, sourcePath, targetPath, spyTargetFs, isSourceLocal, NO_ACID, false, null, false, false, false);
} catch (HiveException e) {
e.printStackTrace();
assertTrue("Hive.copyFiles() threw an unexpected exception.", false);
}
assertTrue(spyTargetFs.exists(new Path(targetPath, "000000_0")));
assertTrue(spyTargetFs.exists(new Path(targetPath, "000001_0")));
assertTrue(spyTargetFs.exists(new Path(targetPath, "000000_0.gz")));
assertTrue(spyTargetFs.exists(new Path(targetPath, "000001_0.gz")));
}
代码示例来源:origin: apache/hive
@Test
public void testRenameNewFilesOnSameFileSystem() throws IOException {
Path sourcePath = new Path(sourceFolder.getRoot().getAbsolutePath());
sourceFolder.newFile("000000_0");
sourceFolder.newFile("000001_0");
sourceFolder.newFile("000000_0.gz");
sourceFolder.newFile("000001_0.gz");
Path targetPath = new Path(targetFolder.getRoot().getAbsolutePath());
FileSystem targetFs = targetPath.getFileSystem(hiveConf);
try {
Hive.copyFiles(hiveConf, sourcePath, targetPath, targetFs, isSourceLocal, NO_ACID, false,null, false, false, false);
} catch (HiveException e) {
e.printStackTrace();
assertTrue("Hive.copyFiles() threw an unexpected exception.", false);
}
assertTrue(targetFs.exists(new Path(targetPath, "000000_0")));
assertTrue(targetFs.exists(new Path(targetPath, "000001_0")));
assertTrue(targetFs.exists(new Path(targetPath, "000000_0.gz")));
assertTrue(targetFs.exists(new Path(targetPath, "000001_0.gz")));
}
代码示例来源:origin: apache/hive
Hive.copyFiles(hiveConf, sourcePath, targetPath, spyTargetFs, isSourceLocal, NO_ACID, false, null, false, false, false);
} catch (HiveException e) {
e.printStackTrace();
Hive.copyFiles(hiveConf, sourcePath, targetPath, spyTargetFs, isSourceLocal, NO_ACID, false, null, false, false, false);
} catch (HiveException e) {
e.printStackTrace();
代码示例来源:origin: apache/hive
Hive.copyFiles(hiveConf, sourcePath, targetPath, targetFs, isSourceLocal, NO_ACID, false, null, false, false, false);
} catch (HiveException e) {
e.printStackTrace();
Hive.copyFiles(hiveConf, sourcePath, targetPath, targetFs, isSourceLocal, NO_ACID, false, null, false, false, false);
} catch (HiveException e) {
e.printStackTrace();
代码示例来源:origin: apache/drill
Hive.copyFiles(conf, loadPath, newPartPath, fs, isSrcLocal, isAcid, newFiles);
代码示例来源:origin: apache/drill
try {
fs = tbl.getDataLocation().getFileSystem(sessionConf);
copyFiles(sessionConf, loadPath, tbl.getPath(), fs, isSrcLocal, isAcid, newFiles);
} catch (IOException e) {
throw new HiveException("addFiles: filesystem error in check phase", e);
代码示例来源:origin: apache/hive
} else {
FileSystem fs = destPath.getFileSystem(conf);
copyFiles(conf, loadPath, destPath, fs, isSrcLocal, isAcidIUDoperation,
(loadFileType == LoadFileType.OVERWRITE_EXISTING), newFiles,
tbl.getNumBuckets() > 0, isFullAcidTable, isManaged);
代码示例来源:origin: apache/hive
try {
FileSystem fs = tbl.getDataLocation().getFileSystem(conf);
copyFiles(conf, loadPath, destPath, fs, isSrcLocal, isAcidIUDoperation,
loadFileType == LoadFileType.OVERWRITE_EXISTING, newFiles,
tbl.getNumBuckets() > 0, isFullAcidTable, isManaged);
代码示例来源:origin: org.apache.hadoop.hive/hive-exec
/**
* Inserts files specified into the partition. Works by moving files
*
* @param srcf
* Files to be moved. Leaf directories or globbed file paths
*/
protected void copyFiles(Path srcf) throws HiveException {
FileSystem fs;
try {
fs = FileSystem.get(getDataLocation(), Hive.get().getConf());
Hive.copyFiles(srcf, new Path(getDataLocation().getPath()), fs);
} catch (IOException e) {
throw new HiveException("addFiles: filesystem error in check phase", e);
}
}
代码示例来源:origin: org.apache.hadoop.hive/hive-exec
} else {
FileSystem fs = FileSystem.get(tbl.getDataLocation(), getConf());
Hive.copyFiles(loadPath, newPartPath, fs);
代码示例来源:origin: com.facebook.presto.hive/hive-apache
newFiles = new ArrayList<Path>();
FileSystem fs = tbl.getDataLocation().getFileSystem(conf);
Hive.copyFiles(conf, loadPath, newPartPath, fs, isSrcLocal, isAcid, newFiles);
代码示例来源:origin: com.facebook.presto.hive/hive-apache
try {
fs = tbl.getDataLocation().getFileSystem(sessionConf);
copyFiles(sessionConf, loadPath, tbl.getPath(), fs, isSrcLocal, isAcid, newFiles);
} catch (IOException e) {
throw new HiveException("addFiles: filesystem error in check phase", e);
内容来源于网络,如有侵权,请联系作者删除!