MapperReduce序列化作業(二)——排序

1資料源 :上次關于手機流量簡單統計業務的product

2.要求  ：根據總流量得值進行倒序排序，然后得到輸出

3.大概邏輯

（1）FlowSort 類：進行序列化和反序列化,排序邏輯介面實作

（2）FlowSortMapper 類：對資料進行封裝

（3）FlowSortReducer類：將key和value進行對調然后封裝寫入（對調位置是因為只有key(形參)是可以排序的，value就不行）

（4）FlowSortDriver類; 進行driver操作（在代碼下面有具體注釋） 這里會把輸出型別的時候把k和value對調回來

4.代碼

（1）FlowSort

package flowsort;

import org.apache.hadoop.io.WritableComparable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

public class FlowSort implements WritableComparable<FlowSort> {
    //上flow流量
    private long upFlow;
    //下flow流量
    private long downFlow;
    //sum流量
    private long sumFlow;
    //空參構造
    public FlowSort() {
    }

    @Override
    public String toString() {
        return upFlow +"\t"
                 +downFlow +"\t"+
                sumFlow
                ;
    }

    //序列化方法
    public void write(DataOutput out) throws IOException {

       out.writeLong(upFlow);
        out.writeLong(downFlow);
        out.writeLong(sumFlow);
    }
    //反序列化方法
    public void readFields(DataInput in) throws IOException {
         this.upFlow=in.readLong();
         this.downFlow=in.readLong();
         this.sumFlow=in.readLong();
    }

    //排序邏輯
    public int compareTo(FlowSort o) {
        int result=0;
        if (sumFlow>o.getSumFlow()) {
            result = -1;
        }else if (sumFlow<o.getSumFlow()){
            result=1;

        }else{
            result=0;        }
        return result;
    }

    public long getUpFlow() {
        return upFlow;
    }

    public void setUpFlow(long upFlow) {
        this.upFlow = upFlow;
    }

    public long getDownFlow() {
        return downFlow;
    }

    public void setDownFlow(long downFlow) {
        this.downFlow = downFlow;
    }

    public long getSumFlow() {
        return sumFlow;
    }

    public void setSumFlow(long sumFlow) {
        this.sumFlow = sumFlow;
    }
}

（2）FlowSortMapper

package flowsort;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class FlowSortMapper extends Mapper<LongWritable, Text,FlowSort,Text> {

    Text k = new Text();
    FlowSort v = new FlowSort();
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

        //獲取一行資料
        String line = value.toString();
        //切割字串
        String[] filds = line.split("\t");
        //封裝物件
        //封裝key
        String phonenumber = filds[0];
        //獲取上行流量
        long upflow=Long.parseLong(filds[1]);
        //獲取下行流量
        long downflow=Long.parseLong(filds[2]);
        //獲取總流量
        long sumflow=Long.parseLong(filds[3]);



        k.set(phonenumber);
        v.setUpFlow(upflow);
        v.setDownFlow(downflow);
        v.setSumFlow(sumflow);

        //寫出
        context.write(v,k);

    }
}

（3）FlowSortReducer

package flowsort;


import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class FlowSortReducer extends Reducer<FlowSort, Text,Text,FlowSort> {
    @Override
    protected void reduce(FlowSort key, Iterable<Text> values, Context context) throws IOException, InterruptedException {


        for (Text phoneNumber : values) {
            context.write(phoneNumber,key);

        }
    }
}

（4）FlowSortdriver

package flowsort;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


import java.io.IOException;

public class FlowSortDrivers {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        //0 封裝輸入輸出路徑
        args =new String[]{"C:/Users/input","C:/Users/output"};
        System.setProperty("hadoop.home.dir","E:/hadoop-2.7.2/");
        //獲取job物件
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);
        //設定jar加載路徑
        job.setJarByClass(FlowSortDrivers.class);
        //關聯mapper和reducer
        job.setMapperClass(FlowSortMapper.class);
        job.setReducerClass(FlowSortReducer.class);
        //設定map最終輸出型別
        job.setMapOutputKeyClass(FlowSort.class);
        job.setMapOutputValueClass(Text.class);
        //設定最終輸出型別
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(FlowSort.class);
        //設定輸入路勁
        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        //提交job
        boolean result = job.waitForCompletion(true);
        System.exit(result ? 0 : 1);

    }
}

5.結果

6. 我的error：路徑沒問題，但是job物件沒執行完，ouput沒有內容

（1）導錯包，這真的是致命

（2）函式名打錯，，，，，

哦，希望以后不要犯這些低級錯誤（小聲嘀咕：以后肯定還會再犯的*3）

2020-05-26

21:30:13

FlowSort

標籤：Java

上一篇：架構設計 | 快取管理模式，監控和記憶體回收策略

下一篇：intellij idea各種設定備忘錄