1、創建一個學生成績的檔案
vi data
2、將txt檔案上傳到hdfs上
hdfs -fs -put /hadoop/data
3、 在eclipse上實作統計學生成績的代碼
（1）ScoreSortEntity.class
（2）ScoreSortReduce.class
package demo2;
import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Reducer;
public class ScoreSortReduce extends Reducer< ScoreSortEntity, Text,Text, ScoreSortEntity> {
@Override
protected void reduce(ScoreSortEntity scoreSortEntity, Iterable<Text> iterable,Context context)
throws IOException, InterruptedException {
try {
context.write(iterable.iterator().next(),scoreSortEntity);
} catch (Exception e) {
Counter countPrint = context.getCounter("Reduce-OutValue",e.getMessage());
countPrint.increment(1l);
}
}
}(3)ScoreSortMapper.class
package demo2;
import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Mapper;
public class ScoreSortMapper extends Mapper<LongWritable, Text, ScoreSortEntity, Text>{
@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String[] fields=value.toString().split("\t");
try {
ScoreSortEntity entity=new ScoreSortEntity(
fields[0],
Integer.parseInt(fields[1]),
Integer.parseInt(fields[2]),
Integer.parseInt(fields[3]),
Integer.parseInt(fields[4]));
context.write(entity, new Text(fields[0]));
} catch (Exception e) {
Counter countPrint = context.getCounter("Map-Exception",e.toString());
countPrint.increment(1l);
}
}
}
(4)ScoreSortDemo.class
package demo2;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.io.Text;
public class ScoreSortDemo {
public static void main(String[] args) throws Exception {
Configuration conf=new Configuration();
Job job=Job.getInstance(conf);
//設定jar
job.setJarByClass(ScoreSortDemo.class);
//設定Map和Reduce類
job.setMapperClass(ScoreSortMapper.class);
job.setReducerClass(ScoreSortReduce.class);
//設定Map輸出
job.setMapOutputKeyClass(ScoreSortEntity.class);
job.setMapOutputValueClass(Text.class);
//設定Reduce輸出
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(ScoreSortEntity.class);
Path inputPath= new Path("/hadoop/data");
Path outputPath = new Path("/hadoop/dataout");
outputPath.getFileSystem(conf).delete(outputPath, true);//如果輸出路徑存在，洗掉之
//設定輸入輸出路徑
FileInputFormat.setInputPaths(job, inputPath);
FileOutputFormat.setOutputPath(job,outputPath);
boolean waitForCompletion= job.waitForCompletion(true);
System.exit(waitForCompletion?0:1);
}
}將寫好的的代碼打架包
可參考(81條訊息) 在eclipse上實作一個 WordCount 程式，并將 WordCount 程式打包發布到 Hadoop 分布式中運行，_柿子鐳的博客-CSDN博客
4、在hadoop中運行
hadoop jar jar_004.jar demo2.ScoreSortDemo查看結果
hadoop fs -cat /hadoop/dataout/part-r-00000 
轉載請註明出處，本文鏈接：https://www.uj5u.com/qita/382834.html
標籤：其他