Hadoop hdfs 基础操作 client Api(1)

1. 初始化连接

@Before
public void setup() throws Exception {
    configuration = new Configuration();
    System.out.println();

    configuration.set("dfs.replication", "1");
    // configuration.set("hadoop.tmp.dir","/home/hadoop0/data/tmp");
    Iterator iterator = configuration.iterator();
    while(iterator.hasNext()
         ){
        System.out.println(iterator.next()); //打印所有配置或默认参数
    }
    fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, HDFS_USER);
    System.out.println("--------------setup-------------");
}

@After
public void teardown(){
    try {
        fileSystem.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
    configuration = null;
    System.out.println("--------------teardown-------------");
}

2. 新增

2.1 创建文件夹

@Test
public void mkDir() throws Exception{
    System.out.println(fileSystem.mkdirs(new Path("/Complete")));
}

2.2 创建文件并写入

@Test
public void create() throws Exception{
    FSDataOutputStream out = fileSystem.create(new Path("/HDFSApp/write/2.txt"));
    out.writeUTF("hello! first wirte to HDFS file!");
    out.flush();
    out.close();
}

2.3 拷贝本地文件到远程

@Test
public void copyFromLocal() throws IOException {
    Path localFile =  new Path("F:/校验大师_2.7.5.1632.zip");
    Path remoteFile = new Path("/HDFSApp/write/校验大师_2.7.5.1632.zip");
    fileSystem.copyFromLocalFile(localFile, remoteFile);
}

2.4 拷贝大文件，带进度条

@Test
public void copyFromLocalLargeFile(){
    String localFile = "F:/校验大师_2.7.5.1632.zip";
    Path remoteFile = new Path("/HDFSApp/write/校验大师_2.7.5.1633.zip");
    InputStream in = null;
    FSDataOutputStream out = null;

    try {
        in = new BufferedInputStream(new FileInputStream(localFile));
        out = fileSystem.create(remoteFile, new Progressable() {
            @Override
            public void progress() {
                System.out.println("!");
            }
        });
    } catch (IOException e) {
        e.printStackTrace();
        System.out.println("创建输入输出流失败！");
    }

    if(in != null && out != null){
        try {
            IOUtils.copyBytes(in, out,2048, true);
        } catch (IOException e) {
            e.printStackTrace();
            System.out.println("上传过程中失败！");
        }
    }
}

3. 查询检索

3.1 读文件内容

@Test
public void text() throws Exception{
    FSDataInputStream in = fileSystem.open(new Path("/output/part-r-00000"));
    IOUtils.copyBytes(in, System.out, 1024);
}

3.2 下载hdfs文件到本地

@Test
public void copyToLocal() throws IOException {
    Path localFile =  new Path("F:/");
    Path remoteFile = new Path("/HDFSApp/write/校验大师_2.7.5.1632.zip");
    fileSystem.copyToLocalFile(remoteFile, localFile);
}

3.3 获取路径下内容

@Test
public void listFiles() throws IOException {
    Path path =  new Path("/HDFSApp/write");
    FileStatus[] files = fileSystem.listStatus(path);
    for (FileStatus file:files) {
        String type = file.isDirectory() ? "文件夹" : "文件";
        String permission = file.getPermission().toString();
        short replication = file.getReplication();
        String filePath = file.getPath().toString();

        System.out.println(type + "\t" + permission + "\t" + replication + "\t" + filePath);
    }
}

3.4 深层遍历路径下文件

@Test
public void listFilesRecursive() throws IOException {
    Path path =  new Path("/");
    RemoteIterator<LocatedFileStatus> remoteIterator = fileSystem.listFiles(path, true);
    while(remoteIterator.hasNext()){
        LocatedFileStatus file = remoteIterator.next();

        String type = file.isDirectory() ? "文件夹" : "文件";
        String permission = file.getPermission().toString();
        short replication = file.getReplication();
        String filePath = file.getPath().toString();

        System.out.println(type + "\t" + permission + "\t" + replication + "\t" + filePath);
    }
}

3.5 获取文件block信息

@Test
public void getBlocksInfo() throws IOException {
    Path path =  new Path("/HDFSApp/write/校验大师_2.7.5.1632.zip");
    FileStatus fileStatus = fileSystem.getFileStatus(path);
    BlockLocation[] blockLocations =  fileSystem.getFileBlockLocations(fileStatus, 0 , fileStatus.getLen());

    for (BlockLocation block: blockLocations) {
        for (String name :block.getNames()) {
            System.out.println(name + ":" + block.getOffset() +"," + block.getLength());
        }
    }
}

4. 修改

4.1 修改文件名

@Test
public void rename() throws IOException {
    Path oPath =  new Path("/HDFSApp/write/0.txt");
    Path nPath = new Path("/HDFSApp/write/3.txt");
    boolean res = fileSystem.rename(oPath, nPath);
    System.out.println(res);
}

5. 删除

5.1 删除文件或目录

public void delete() throws IOException {
    Path path =  new Path("/HDFSApp/write/校验大师_2.7.5.1632.zip");
    boolean res = fileSystem.delete(path, false);
    System.out.println(res);
}