一.示例
Java抽象类org.apache.hadoop.fs.FileSystem定义了hadoop的一个文件系统接口。Hadoop中关于文件操作类基本上全部是在"org.apache.hadoop.fs"包中，这些API能够支持的操作包含：打开文件，读写文件，删除文件等。
Hadoop类库中最终面向用户提供的接口类是FileSystem，该类是个抽象类，只能通过来类的get方法得到具体类。
public class HDFSFileSystem {
public static void main(String[] args) {
String res = args[0];
long start,end;
start = System.currentTimeMillis();
FileSystem fs = init();
//String res="/tmp/linkis/hdfs/20201215/6e8fa663-ffa8-4d50-9510-f8cf15518dc0.json";
try {
FSDataInputStream in = fs.open(new Path(res));
IOUtils.copyBytes(in, System.out, 4096, false);
} catch (IOException e) {
e.printStackTrace();
}
end = System.currentTimeMillis();
System.out.println("\n start time:" + start+ "; end time:" + end+ "; Run Time:" + (end - start) + "(ms)");
}
public static FileSystem init(){
Configuration conf = new Configuration();
String hadoopConfDir="/etc/hadoop/conf";
conf.addResource(new Path(Paths.get(hadoopConfDir,"core-site.xml").toAbsolutePath().toFile().getAbsolutePath()));
conf.addResource(new Path(Paths.get(hadoopConfDir, "hdfs-site.xml").toAbsolutePath().toFile().getAbsolutePath()));
conf.addResource(new Path(Paths.get(hadoopConfDir, "yarn-site.xml").toAbsolutePath().toFile().getAbsolutePath()));
conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
try {
FileSystem fs = UserGroupInformation.createRemoteUser("hdfs").doAs(new PrivilegedExceptionAction<FileSystem>() {
@Override
public FileSystem run() throws Exception {
return FileSystem.get(conf);
}
});
return fs;
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
return null;
}
}
结果
公众号
