大数据存储技术（2）—— HDFS分布式文件系统

public static FileSystem.get(Configuration conf) throws IOException
public static FileSystem.get(URI uri, Configuration conf) throws IOException

[root@bigdata zhc]# hdfs dfs
Usage: hadoop fs [generic options]
  [-appendToFile <localsrc> ... <dst>]
  [-cat [-ignoreCrc] <src> ...]
  [-checksum <src> ...]
  [-chgrp [-R] GROUP PATH...]
  [-chmod [-R] <MODE[,MODE]... | OCTALMODE> PATH...]
  [-chown [-R] [OWNER][:[GROUP]] PATH...]
  [-copyFromLocal [-f] [-p] [-l] [-d] [-t <thread count>] <localsrc> ... <dst>]
  [-copyToLocal [-f] [-p] [-ignoreCrc] [-crc] <src> ... <localdst>]
  [-count [-q] [-h] [-v] [-t [<storage type>]] [-u] [-x] [-e] <path> ...]
  [-cp [-f] [-p | -p[topax]] [-d] <src> ... <dst>]
  [-createSnapshot <snapshotDir> [<snapshotName>]]
  [-deleteSnapshot <snapshotDir> <snapshotName>]
  [-df [-h] [<path> ...]]
  [-du [-s] [-h] [-v] [-x] <path> ...]
  [-expunge]
  [-find <path> ... <expression> ...]
  [-get [-f] [-p] [-ignoreCrc] [-crc] <src> ... <localdst>]
  [-getfacl [-R] <path>]
  [-getfattr [-R] {-n name | -d} [-e en] <path>]
  [-getmerge [-nl] [-skip-empty-file] <src> <localdst>]
  [-head <file>]
  [-help [cmd ...]]
  [-ls [-C] [-d] [-h] [-q] [-R] [-t] [-S] [-r] [-u] [-e] [<path> ...]]
  [-mkdir [-p] <path> ...]
  [-moveFromLocal <localsrc> ... <dst>]
  [-moveToLocal <src> <localdst>]
  [-mv <src> ... <dst>]
  [-put [-f] [-p] [-l] [-d] <localsrc> ... <dst>]
  [-renameSnapshot <snapshotDir> <oldName> <newName>]
  [-rm [-f] [-r|-R] [-skipTrash] [-safely] <src> ...]
  [-rmdir [--ignore-fail-on-non-empty] <dir> ...]
  [-setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>]]
  [-setfattr {-n name [-v value] | -x name} <path>]
  [-setrep [-R] [-w] <rep> <path> ...]
  [-stat [format] <path> ...]
  [-tail [-f] [-s <sleep interval>] <file>]
  [-test -[defsz] <path>]
  [-text [-ignoreCrc] <src> ...]
  [-touch [-a] [-m] [-t TIMESTAMP ] [-c] <path> ...]
  [-touchz <path> ...]
  [-truncate [-w] <length> <path> ...]
  [-usage [cmd ...]]
 
Generic options supported are:
-conf <configuration file>        specify an application configuration file
-D <property=value>               define a value for a given property
-fs <file:///|hdfs://namenode:port> specify default filesystem URL to use, overrides 'fs.defaultFS' property from configurations.
-jt <local|resourcemanager:port>  specify a ResourceManager
-files <file1,...>                specify a comma-separated list of files to be copied to the map reduce cluster
-libjars <jar1,...>               specify a comma-separated list of jar files to be included in the classpath
-archives <archive1,...>          specify a comma-separated list of archives to be unarchived on the compute machines
 
The general command line syntax is:
command [genericOptions] [commandOptions]
 

1. [root@bigdata zhc]# start-dfs.sh
2. [root@bigdata zhc]# start-yarn.sh

[root@bigdata zhc]# hdfs dfs -help rm

[root@bigdata zhc]# hdfs dfs -ls /

[root@bigdata zhc]# hdfs dfs -mkdir -p /sanguo/shuguo/

[root@bigdata zhc]# touch kongming.txt
[root@bigdata zhc]# hdfs dfs -moveFromLocal ./kongming.txt /sanguo/shuguo

[root@bigdata zhc]# touch liubei.txt
[root@bigdata zhc]# vi liubei.txt
输入
san gu mao lu
[root@bigdata zhc]# hdfs dfs -appendToFile liubei.txt /sanguo/shuguo/kongming.txt

[root@bigdata zhc]# hdfs dfs -cat /sanguo/shuguo/kongming.txt

[root@bigdata zhc]# hdfs dfs -chmod 666 /sanguo/shuguo/kongming.txt
[root@bigdata zhc]# hdfs dfs -chown atguigu:atguigu /sanguo/shuguo/kongming.txt

[root@bigdata zhc]# hdfs dfs -copyFromLocal README.txt /

[root@bigdata zhc]# hdfs dfs -copyToLocal /sanguo/shuguo/kongming.txt ./

[root@bigdata zhc]# hdfs dfs -cp /sanguo/shuguo/kongming.txt /zhuge.txt

[root@bigdata zhc]# hdfs dfs -mv /zhuge.txt /sanguo/shuguo/

[root@bigdata zhc]# hdfs dfs -get /sanguo/shuguo/kongming.txt ./

[root@bigdata zhc]# hdfs dfs -getmerge /user/atguigu/test/* ./zaiyiqi.txt

[root@bigdata zhc]# hdfs dfs -put ./zaiyiqi.txt /user/atguigu/test/

[root@bigdata zhc]# hdfs dfs -tail /sanguo/shuguo/kongming.txt

[root@bigdata zhc]# hdfs dfs -rm /user/atguigu/test/jinlian2.txt

1. [root@bigdata zhc]# hdfs dfs -mkdir /test
2. [root@bigdata zhc]# hdfs dfs -rmdir /test

[root@bigdata zhc]# hdfs dfs -du -s -h /user/atguigu/test
2.7 K /user/atguigu/test
[root@bigdata zhc]# hdfs dfs -du -h /user/atguigu/test
1.3 K /user/atguigu/test/README.txt
15 /user/atguigu/test/jinlian.txt
1.4 K /user/atguigu/test/zaiyiqi.txt

[root@bigdata zhc]# hdfs dfs -setrep 10 /sanguo/shuguo/kongming.txt

@Test
public void testCopyFromLocalFile() throws IOException,InterruptedException,URISyntaxException {
    // 1 获取文件系统
    Configuration configuration = new Configuration();
    configuration.set("dfs.replication", "2");
    FileSystem fs = FileSystem.get(new URI("hdfs://bigdata:9000"),configuration,"atguigu");
    // 2 上传文件
    fs.copyFromLocalFile(new Path("e:/banzhang.txt"), new Path("/banzhang.txt"));
    // 3 关闭资源
    fs.close();
    System.out.println("over");
}

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
  <property>
    <name>dfs.replication</name>
      <value>1</value>
  </property>
</configuration>

@Test
public void testCopyToLocalFile() throws IOException,InterruptedException,URISyntaxException{
    // 1 获取文件系统
    Configuration configuration = new Configuration();
    FileSystem fs = FileSystem.get(new URI("hdfs://bigdata:9000"), configuration, "atguigu");
    // 2 执行下载操作
    // boolean delSrc 指是否将原文件删除
    // Path src 指要下载的文件路径
    // Path dst 指将文件下载到的路径
    // boolean useRawLocalFileSystem 是否开启文件校验
    fs.copyToLocalFile(false, new Path("/banzhang.txt"), new Path("e:/banhua.txt"), true);
    // 3 关闭资源
    fs.close();
}

@Test
public void testDelete() throws IOException, InterruptedException, URISyntaxException{
    // 1 获取文件系统
    Configuration configuration = new Configuration();
    FileSystem fs = FileSystem.get(new URI("hdfs://bigdata:9000"), configuration, "atguigu");
    // 2 执行删除
    fs.delete(new Path("/0508/"), true);
    // 3 关闭资源
    fs.close();
}

@Test
public void testRename() throws IOException, InterruptedException, URISyntaxException{
    // 1 获取文件系统
    Configuration configuration = new Configuration();
    FileSystem fs = FileSystem.get(new URI("hdfs://bigdata:9000"), configuration, "atguigu"); 
    // 2 修改文件名称
    fs.rename(new Path("/banzhang.txt"), new Path("/banhua.txt"));
    // 3 关闭资源
    fs.close();
}

@Test
public void testListFiles() throws IOException, InterruptedException, URISyntaxException{
    // 1 获取文件系统
    Configuration configuration = new Configuration();
    FileSystem fs = FileSystem.get(new URI("hdfs://bigdata:9000"), configuration, "atguigu"); 
    // 2 获取文件详情
    RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);
    while(listFiles.hasNext()){
        LocatedFileStatus status = listFiles.next();
        // 输出详情
        // 文件名称
        System.out.println(status.getPath().getName());
        // 长度
        System.out.println(status.getLen());
        // 权限
        System.out.println(status.getPermission());
        // 分组
        System.out.println(status.getGroup());
        // 获取存储的块信息
        BlockLocation[] blockLocations = status.getBlockLocations();
        for (BlockLocation blockLocation : blockLocations) {
            // 获取块存储的主机节点
            String[] hosts = blockLocation.getHosts();
            for (String host : hosts) {
                System.out.println(host);
            }
        }
        System.out.println("-----------班长的分割线----------");
    }
    // 3 关闭资源
    fs.close();
}

@Test
public void testListStatus() throws IOException, InterruptedException, URISyntaxException{
    // 1 获取文件配置信息
    Configuration configuration = new Configuration();
    FileSystem fs =FileSystem.get(newURI("hdfs://bigdata:9000"),configuration,"atguigu");
    // 2 判断是文件还是文件夹
    FileStatus[] listStatus = fs.listStatus(new Path("/"));
    for (FileStatus fileStatus : listStatus) {
        // 如果是文件
        if (fileStatus.isFile()) {
            System.out.println("f:"+fileStatus.getPath().getName());
        }else {
            System.out.println("d:"+fileStatus.getPath().getName());
        }
    }
    // 3 关闭资源
    fs.close();
}