Java實作MapReduce Wordcount案例

先改pom.xml：

<project xmlns="http://maven.apache.org/POM/4.0.0"	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">	<modelVersion>4.0.0</modelVersion>	<groupId>com.mcq</groupId>	<artifactId>mr-1101</artifactId>	<version>0.0.1-SNAPSHOT</version>	<dependencies>		<dependency>			<groupId>jdk.tools</groupId>			<artifactId>jdk.tools</artifactId>			<version>1.8</version>			<scope>system</scope>			<systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>		</dependency>		<dependency>			<groupId>junit</groupId>			<artifactId>junit</artifactId>			<version>RELEASE</version>		</dependency>		<dependency>			<groupId>org.apache.logging.log4j</groupId>			<artifactId>log4j-core</artifactId>			<version>2.8.2</version>		</dependency>		<dependency>			<groupId>org.apache.hadoop</groupId>			<artifactId>hadoop-common</artifactId>			<version>2.7.2</version>		</dependency>		<dependency>			<groupId>org.apache.hadoop</groupId>			<artifactId>hadoop-client</artifactId>			<version>2.7.2</version>		</dependency>		<dependency>			<groupId>org.apache.hadoop</groupId>			<artifactId>hadoop-hdfs</artifactId>			<version>2.7.2</version>		</dependency>	</dependencies></project>

在resources檔案夾下添加檔案 log4j.properties：

log4j.rootLogger=INFO, stdoutlog4j.appender.stdout=org.apache.log4j.ConsoleAppenderlog4j.appender.stdout.layout=org.apache.log4j.PatternLayoutlog4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%nlog4j.appender.logfile=org.apache.log4j.FileAppenderlog4j.appender.logfile.File=target/spring.loglog4j.appender.logfile.layout=org.apache.log4j.PatternLayoutlog4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n

 WordcountDriver.java：

package com.mcq;import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public class WordcountDriver{	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {		System.out.println("hello");		Configuration conf=new Configuration();		//1.獲取Job物件		Job job=Job.getInstance(conf);		//2.設定jar存盤位置		job.setJarByClass(WordcountDriver.class);		//3.關聯Map和Reduce類		job.setMapperClass(WordcountMapper.class);		job.setReducerClass(WordcountReducer.class);		//4.設定Mapper階段輸出資料的key和value型別		job.setMapOutputKeyClass(Text.class);		job.setMapOutputValueClass(IntWritable.class);		//5.設定最終輸出的key和value型別		job.setOutputKeyClass(Text.class);		job.setOutputValueClass(IntWritable.class);		//6.設定輸入路徑和輸出路徑		FileInputFormat.setInputPaths(job, new Path(args[0]));		FileOutputFormat.setOutputPath(job, new Path(args[1]));		//7.提交Job//		job.submit();		job.waitForCompletion(true);//		boolean res=job.waitForCompletion(true);//true表示列印結果//		System.exit(res?0:1);	}}

 WordcountMapper.java：

package com.mcq;import java.io.IOException;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Mapper;//map階段//KEYIN:輸入資料的key（偏移量，比如第一行是0~19，第二行是20~25），必須是LongWritable//VALUEIN:輸入資料的value（比如文本內容是字串，那就填Text）//KEYOUT:輸出資料的key型別//VALUEOUT:輸出資料的值型別public class WordcountMapper extends Mapper<LongWritable, Text, Text, IntWritable>{	IntWritable v=new IntWritable(1);	Text k = new Text();	@Override	protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, IntWritable>.Context context)			throws IOException, InterruptedException {		// TODO Auto-generated method stub		//1.獲取一行		String line=value.toString();		//2.切割單詞		String[] words=line.split(" ");		//3.回圈寫出		for(String word:words) {			k.set(word);			context.write(k, v);		}	}}

 WordcountReducer.java：

package com.mcq;import java.io.IOException;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Reducer;//KEYIN、VALUEIN：map階段輸出的key和value型別public class WordcountReducer extends Reducer<Text, IntWritable, Text, IntWritable>{	IntWritable v=new IntWritable();	@Override	protected void reduce(Text key, Iterable<IntWritable> values,			Reducer<Text, IntWritable, Text, IntWritable>.Context context) throws IOException, InterruptedException {		// TODO Auto-generated method stub		int sum=0;		for(IntWritable value:values) {			sum+=value.get();		}		v.set(sum);		context.write(key, v);	}}

在run configuration里加上引數e:/mrtest/in.txt e:/mrtest/out.txt

運行時遇到了個bug，參考https://blog.csdn.net/qq_40310148/article/details/86617512解決了

在集群上運行：

用maven打成jar包，需要添加一些打包依賴：

	<build>		<plugins>			<plugin>				<artifactId>maven-compiler-plugin</artifactId>				<version>2.3.2</version>				<configuration>					<source>1.8</source>					<target>1.8</target>				</configuration>			</plugin>			<plugin>				<artifactId>maven-assembly-plugin </artifactId>				<configuration>					<descriptorRefs>						<descriptorRef>jar-with-dependencies</descriptorRef>					</descriptorRefs>					<archive>						<manifest>							<mainClass>com.mcq.WordcountDriver</mainClass>						</manifest>					</archive>				</configuration>				<executions>					<execution>						<id>make-assembly</id>						<phase>package</phase>						<goals>							<goal>single</goal>						</goals>					</execution>				</executions>			</plugin>		</plugins>	</build>

 注意上面mainClass里要填驅動類的主類名，可以點擊類名右鍵copy qualified name，

將程式打成jar包（具體操作：右鍵工程名run as maven install，然后target檔案夾會產生兩個jar包，我們把不用依賴的包拷貝到hadoop集群上，因為集群已經配好相關依賴了），上傳到集群

輸入以下命令運行

hadoop jar mr-1101-0.0.1-SNAPSHOT.jar com.mcq.WordcountDriver /xiaocao.txt /output

注意這里輸入輸出的路徑是集群上的路徑，

標籤：大數據

上一篇：大資料寫圖書館行為分析

下一篇：oracle12C連接問題