spark-1.6.1-bin-hadoop2.6里Basic包下的JavaPageRank.java
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ //package org.apache.spark.examples; package zhouls.bigdata.Basic; import scala.Tuple2;//scala里的元组 import com.google.common.collect.Iterables; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.Function; import org.apache.spark.api.java.function.Function2; import org.apache.spark.api.java.function.PairFlatMapFunction; import org.apache.spark.api.java.function.PairFunction; import java.util.ArrayList; import java.util.List; import java.util.Iterator; import java.util.regex.Pattern; /** * Computes the PageRank of URLs from an input file. Input file should * be in format of: * URL neighbor URL * URL neighbor URL * URL neighbor URL * ... * where URL and their neighbors are separated by space(s). * * This is an example implementation for learning how to use Spark. For more conventional use, * please refer to org.apache.spark.graphx.lib.PageRank */ public final class JavaPageRank { private static final Pattern SPACES = Pattern.compile("\\s+"); /* * 显示警告函数 */ static void showWarning() { String warning = "WARN: This is a naive implementation of PageRank " + "and is given as an example! \n" + "Please use the PageRank implementation found in " + "org.apache.spark.graphx.lib.PageRank for more conventional use."; System.err.println(warning); } private static class Sum implements Function2<Double, Double, Double> { @Override public Double call(Double a, Double b) { return a + b; } } /* * 主函数 */ public static void main(String[] args) throws Exception { if (args.length < 2) { System.err.println("Usage: JavaPageRank <file> <number_of_iterations>"); System.exit(1); } showWarning(); SparkConf sparkConf = new SparkConf().setAppName("JavaPageRank").setMaster("local"); JavaSparkContext ctx = new JavaSparkContext(sparkConf); // Loads in input file. It should be in format of: // URL neighbor URL // URL neighbor URL // URL neighbor URL // ... // JavaRDD<String> lines = ctx.textFile(args[0], 1);//这是官网发行包里写的 JavaRDD<String> lines = ctx.textFile("data/input/mllib/pagerank_data.txt", 1); // Loads all URLs from input file and initialize their neighbors. //根据边关系数据生成 邻接表 如：(1,(2,3,4,5)) (2,(1,5))... JavaPairRDD<String, Iterable<String>> links = lines.mapToPair(new PairFunction<String, String, String>() { @Override public Tuple2<String, String> call(String s) { String[] parts = SPACES.split(s); return new Tuple2<String, String>(parts[0], parts[1]); } }).distinct().groupByKey().cache(); //初始化 ranks, 每一个url初始分值为1 // Loads all URLs with other URL(s) link to from input file and initialize ranks of them to one. JavaPairRDD<String, Double> ranks = links.mapValues(new Function<Iterable<String>, Double>() { @Override public Double call(Iterable<String> rs) { return 1.0; } }); /* * 迭代iters次； 每次迭代中做如下处理， links（urlKey, neighborUrls） join (urlKey, rank(分值))； * 对neighborUrls以及初始 rank，每一个neighborUrl , neighborUrlKey, 初始rank/size(新的rank贡献值)； * 然后再进行reduceByKey相加 并对分值 做调整 0.15 + 0.85 * _ */ // Calculates and updates URL ranks continuously using PageRank algorithm. for (int current = 0; current < Integer.parseInt(args[1]); current++) { // Calculates URL contributions to the rank of other URLs. JavaPairRDD<String, Double> contribs = links.join(ranks).values() .flatMapToPair(new PairFlatMapFunction<Tuple2<Iterable<String>, Double>, String, Double>() { @Override public Iterable<Tuple2<String, Double>> call(Tuple2<Iterable<String>, Double> s) { int urlCount = Iterables.size(s._1); List<Tuple2<String, Double>> results = new ArrayList<Tuple2<String, Double>>(); for (String n : s._1) { results.add(new Tuple2<String, Double>(n, s._2() / urlCount)); } return results; } }); // Re-calculates URL ranks based on neighbor contributions. ranks = contribs.reduceByKey(new Sum()).mapValues(new Function<Double, Double>() { @Override public Double call(Double sum) { return 0.15 + sum * 0.85; } }); } //输出排名 // Collects all URL ranks and dump them to console. List<Tuple2<String, Double>> output = ranks.collect(); for (Tuple2<?,?> tuple : output) { System.out.println(tuple._1() + " has rank: " + tuple._2() + "."); } ctx.stop(); } }
没结果，暂时
spark-2.2.0-bin-hadoop2.6里Basic包下的JavaPageRank.java
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ //package org.apache.spark.examples; package zhouls.bigdata.Basic; import java.util.ArrayList; import java.util.List; import java.util.regex.Pattern; import scala.Tuple2; import com.google.common.collect.Iterables; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.function.Function2; import org.apache.spark.sql.SparkSession; /** * Computes the PageRank of URLs from an input file. Input file should * be in format of: * URL neighbor URL * URL neighbor URL * URL neighbor URL * ... * where URL and their neighbors are separated by space(s). * * This is an example implementation for learning how to use Spark. For more conventional use, * please refer to org.apache.spark.graphx.lib.PageRank * * Example Usage: * <pre> * bin/run-example JavaPageRank data/mllib/pagerank_data.txt 10 * </pre> */ public final class JavaPageRank { private static final Pattern SPACES = Pattern.compile("\\s+"); /* * 显示警告函数 */ static void showWarning() { String warning = "WARN: This is a naive implementation of PageRank " + "and is given as an example! \n" + "Please use the PageRank implementation found in " + "org.apache.spark.graphx.lib.PageRank for more conventional use."; System.err.println(warning); } private static class Sum implements Function2<Double, Double, Double> { @Override public Double call(Double a, Double b) { return a + b; } } /* * 主函数 */ public static void main(String[] args) throws Exception { if (args.length < 2) { System.err.println("Usage: JavaPageRank <file> <number_of_iterations>"); System.exit(1); } showWarning(); SparkSession spark = SparkSession .builder() .master("local") .appName("JavaPageRank") .getOrCreate(); // Loads in input file. It should be in format of: // URL neighbor URL // URL neighbor URL // URL neighbor URL // ... // JavaRDD<String> lines = spark.read().textFile(args[0]).javaRDD(); JavaRDD<String> lines = spark.read().textFile("data/input/mllib/pagerank_data.txt").javaRDD(); // Loads all URLs from input file and initialize their neighbors. //根据边关系数据生成 邻接表 如：(1,(2,3,4,5)) (2,(1,5))... JavaPairRDD<String, Iterable<String>> links = lines.mapToPair(s -> { String[] parts = SPACES.split(s); return new Tuple2<>(parts[0], parts[1]); }).distinct().groupByKey().cache(); // Loads all URLs with other URL(s) link to from input file and initialize ranks of them to one. //初始化 ranks, 每一个url初始分值为1 JavaPairRDD<String, Double> ranks = links.mapValues(rs -> 1.0); /* * 迭代iters次； 每次迭代中做如下处理， links（urlKey, neighborUrls） join (urlKey, rank(分值))； * 对neighborUrls以及初始 rank，每一个neighborUrl , neighborUrlKey, 初始rank/size(新的rank贡献值)； * 然后再进行reduceByKey相加 并对分值 做调整 0.15 + 0.85 * _ */ // Calculates and updates URL ranks continuously using PageRank algorithm. for (int current = 0; current < Integer.parseInt(args[1]); current++) { // Calculates URL contributions to the rank of other URLs. JavaPairRDD<String, Double> contribs = links.join(ranks).values() .flatMapToPair(s -> { int urlCount = Iterables.size(s._1()); List<Tuple2<String, Double>> results = new ArrayList<>(); for (String n : s._1) { results.add(new Tuple2<>(n, s._2() / urlCount)); } return results.iterator(); }); // Re-calculates URL ranks based on neighbor contributions. ranks = contribs.reduceByKey(new Sum()).mapValues(sum -> 0.15 + sum * 0.85); } //输出排名 // Collects all URL ranks and dump them to console. List<Tuple2<String, Double>> output = ranks.collect(); for (Tuple2<?,?> tuple : output) { System.out.println(tuple._1() + " has rank: " + tuple._2() + "."); } spark.stop(); } }
本文转自大数据躺过的坑博客园博客，原文链接：http://www.cnblogs.com/zlslch/p/7458368.html，如需转载请自行联系原作者
