hadoop: hdfs API示例

  1 import org.apache.hadoop.conf.Configuration;
  2 import org.apache.hadoop.fs.*;
  3 import org.apache.hadoop.io.IOUtils;
  4 
  5 import java.io.ByteArrayOutputStream;
  6 import java.io.IOException;
  7 import java.io.InputStream;
  8 import java.io.OutputStream;
  9 
 10 /**
 11  * HDFS工具类
 12  * Author: 菩提树下的杨过(http://yjmyzz.cnblogs.com)
 13  * Since: 2015-05-21
 14  */
 15 public class HDFSUtil {
 16 
 17 
 18     private HDFSUtil() {
 19 
 20     }
 21 
 22     /**
 23      * 判断路径是否存在
 24      *
 25      * @param conf
 26      * @param path
 27      * @return
 28      * @throws IOException
 29      */
 30     public static boolean exits(Configuration conf, String path) throws IOException {
 31         FileSystem fs = FileSystem.get(conf);
 32         return fs.exists(new Path(path));
 33     }
 34 
 35     /**
 36      * 创建文件
 37      *
 38      * @param conf
 39      * @param filePath
 40      * @param contents
 41      * @throws IOException
 42      */
 43     public static void createFile(Configuration conf, String filePath, byte[] contents) throws IOException {
 44         FileSystem fs = FileSystem.get(conf);
 45         Path path = new Path(filePath);
 46         FSDataOutputStream outputStream = fs.create(path);
 47         outputStream.write(contents);
 48         outputStream.close();
 49         fs.close();
 50     }
 51 
 52     /**
 53      * 创建文件
 54      *
 55      * @param conf
 56      * @param filePath
 57      * @param fileContent
 58      * @throws IOException
 59      */
 60     public static void createFile(Configuration conf, String filePath, String fileContent) throws IOException {
 61         createFile(conf, filePath, fileContent.getBytes());
 62     }
 63 
 64     /**
 65      * @param conf
 66      * @param localFilePath
 67      * @param remoteFilePath
 68      * @throws IOException
 69      */
 70     public static void copyFromLocalFile(Configuration conf, String localFilePath, String remoteFilePath) throws IOException {
 71         FileSystem fs = FileSystem.get(conf);
 72         Path localPath = new Path(localFilePath);
 73         Path remotePath = new Path(remoteFilePath);
 74         fs.copyFromLocalFile(true, true, localPath, remotePath);
 75         fs.close();
 76     }
 77 
 78     /**
 79      * 删除目录或文件
 80      *
 81      * @param conf
 82      * @param remoteFilePath
 83      * @param recursive
 84      * @return
 85      * @throws IOException
 86      */
 87     public static boolean deleteFile(Configuration conf, String remoteFilePath, boolean recursive) throws IOException {
 88         FileSystem fs = FileSystem.get(conf);
 89         boolean result = fs.delete(new Path(remoteFilePath), recursive);
 90         fs.close();
 91         return result;
 92     }
 93 
 94     /**
 95      * 删除目录或文件(如果有子目录,则级联删除)
 96      *
 97      * @param conf
 98      * @param remoteFilePath
 99      * @return
100      * @throws IOException
101      */
102     public static boolean deleteFile(Configuration conf, String remoteFilePath) throws IOException {
103         return deleteFile(conf, remoteFilePath, true);
104     }
105 
106     /**
107      * 文件重命名
108      *
109      * @param conf
110      * @param oldFileName
111      * @param newFileName
112      * @return
113      * @throws IOException
114      */
115     public static boolean renameFile(Configuration conf, String oldFileName, String newFileName) throws IOException {
116         FileSystem fs = FileSystem.get(conf);
117         Path oldPath = new Path(oldFileName);
118         Path newPath = new Path(newFileName);
119         boolean result = fs.rename(oldPath, newPath);
120         fs.close();
121         return result;
122     }
123 
124     /**
125      * 创建目录
126      *
127      * @param conf
128      * @param dirName
129      * @return
130      * @throws IOException
131      */
132     public static boolean createDirectory(Configuration conf, String dirName) throws IOException {
133         FileSystem fs = FileSystem.get(conf);
134         Path dir = new Path(dirName);
135         boolean result = fs.mkdirs(dir);
136         fs.close();
137         return result;
138     }
139 
140     /**
141      * 列出指定路径下的所有文件(不包含目录)
142      *
143      * @param conf
144      * @param basePath
145      * @param recursive
146      */
147     public static RemoteIterator<LocatedFileStatus> listFiles(FileSystem fs, String basePath, boolean recursive) throws IOException {
148 
149         RemoteIterator<LocatedFileStatus> fileStatusRemoteIterator = fs.listFiles(new Path(basePath), recursive);
150 
151         return fileStatusRemoteIterator;
152     }
153 
154     /**
155      * 列出指定路径下的文件（非递归）
156      *
157      * @param conf
158      * @param basePath
159      * @return
160      * @throws IOException
161      */
162     public static RemoteIterator<LocatedFileStatus> listFiles(Configuration conf, String basePath) throws IOException {
163         FileSystem fs = FileSystem.get(conf);
164         RemoteIterator<LocatedFileStatus> remoteIterator = fs.listFiles(new Path(basePath), false);
165         fs.close();
166         return remoteIterator;
167     }
168 
169     /**
170      * 列出指定目录下的文件\子目录信息（非递归）
171      *
172      * @param conf
173      * @param dirPath
174      * @return
175      * @throws IOException
176      */
177     public static FileStatus[] listStatus(Configuration conf, String dirPath) throws IOException {
178         FileSystem fs = FileSystem.get(conf);
179         FileStatus[] fileStatuses = fs.listStatus(new Path(dirPath));
180         fs.close();
181         return fileStatuses;
182     }
183 
184 
185     /**
186      * 读取文件内容
187      *
188      * @param conf
189      * @param filePath
190      * @return
191      * @throws IOException
192      */
193     public static String readFile(Configuration conf, String filePath) throws IOException {
194         String fileContent = null;
195         FileSystem fs = FileSystem.get(conf);
196         Path path = new Path(filePath);
197         InputStream inputStream = null;
198         ByteArrayOutputStream outputStream = null;
199         try {
200             inputStream = fs.open(path);
201             outputStream = new ByteArrayOutputStream(inputStream.available());
202             IOUtils.copyBytes(inputStream, outputStream, conf);
203             fileContent = outputStream.toString();
204         } finally {
205             IOUtils.closeStream(inputStream);
206             IOUtils.closeStream(outputStream);
207             fs.close();
208         }
209         return fileContent;
210     }
211 }

 1     @Test
 2     public void test() throws IOException {
 3         Configuration conf = new Configuration();
 4         String newDir = "/test";
 5         //01.检测路径是否存在 测试
 6         if (HDFSUtil.exits(conf, newDir)) {
 7             System.out.println(newDir + " 已存在!");
 8         } else {
 9             //02.创建目录测试
10             boolean result = HDFSUtil.createDirectory(conf, newDir);
11             if (result) {
12                 System.out.println(newDir + " 创建成功!");
13             } else {
14                 System.out.println(newDir + " 创建失败!");
15             }
16         }
17         String fileContent = "Hi,hadoop. I love you";
18         String newFileName = newDir + "/myfile.txt";
19 
20         //03.创建文件测试
21         HDFSUtil.createFile(conf, newFileName, fileContent);
22         System.out.println(newFileName + " 创建成功");
23 
24         //04.读取文件内容 测试
25         System.out.println(newFileName + " 的内容为:\n" + HDFSUtil.readFile(conf, newFileName));
26 
27         //05. 测试获取所有目录信息
28         FileStatus[] dirs = HDFSUtil.listStatus(conf, "/");
29         System.out.println("--根目录下的所有子目录---");
30         for (FileStatus s : dirs) {
31             System.out.println(s);
32         }
33 
34         //06. 测试获取所有文件
35         FileSystem fs = FileSystem.get(conf);
36         RemoteIterator<LocatedFileStatus> files = HDFSUtil.listFiles(fs, "/", true);
37         System.out.println("--根目录下的所有文件---");
38         while (files.hasNext()) {
39             System.out.println(files.next());
40         }
41         fs.close();
42 
43         //删除文件测试
44         boolean isDeleted = HDFSUtil.deleteFile(conf, newDir);
45         System.out.println(newDir + " 已被删除");
46 
47     }