WordCount程式撰寫
Java
public class WordCount {
public static void main(String[] args) {
// 配置sparkconf，啟動spark應用程式
SparkConf sparkConf = new SparkConf().setAppName("spark").setMaster("local");
JavaSparkContext sc = new JavaSparkContext(sparkConf);
//使用textFile方法讀取檔案，逐行讀取
JavaRDD<String> line = sc.textFile("D:/JavaCode/javascala/src/main/scala/rdd/builder/word");
// flatMap算子，拆分單詞
JavaRDD<String> words = line.flatMap(new FlatMapFunction<String, String>() {
public Iterator<String> call(String s) throws Exception {
return Arrays.asList(s.split(" ")).iterator();
}
});
//map算子，對單詞進行映射 word =>（Word，1）
JavaPairRDD<String, Integer> word = words.mapToPair(new PairFunction<String, String, Integer>() {
public Tuple2<String, Integer> call(String s) throws Exception {
return new Tuple2<String, Integer>(s, 1);
}
});
//reduceByKey算子，最相同的單詞進行分組聚合
JavaPairRDD<String, Integer> wordCount = word.reduceByKey(new Function2<Integer, Integer, Integer>() {
public Integer call(Integer v1, Integer v2) throws Exception {
return v1 + v2;
}
});
//遍歷列印結果
wordCount.foreach(new VoidFunction<Tuple2<String, Integer>>() {
public void call(Tuple2<String, Integer> stringIntegerTuple2) throws Exception {
System.out.println(stringIntegerTuple2._1+":"+stringIntegerTuple2._2);
}
});
sc.stop();
}
}
Scala
object WordCount {
def main(args: Array[String]): Unit = {
//wordcount程式
//1.創建spark應用程式
val sparkConf = new SparkConf().setMaster("local[*]").setAppName("WordCount")
val sc = new SparkContext(sparkConf)
//2. 讀取檔案
val words = sc.textFile("D:/JavaCode/javascala/src/main/scala/rdd/builder/word")
//3. 分詞
val word = words.flatMap(_.split(" "))
//4.map映射
val word_map = word.map(x => (x, 1))
//5.聚合
val word_count = word_map.reduceByKey(_ + _)
//6.列印結果
word_count.sortBy((x)=>(-x._2)).collect().foreach(println)
word_count.groupByKey().foreach(println)
//7.關閉spark程式
sc.stop()
}
}
總結：使用Scala開發spark應用程式，相對Java要更簡潔，更方便，
轉載請註明出處，本文鏈接：https://www.uj5u.com/qita/419469.html
標籤：其他