在线午夜精品自拍小视频_无码av无码专区线_亚洲无码精品人妻_人人澡欧美一区

<!-- 版本信息 -->
    <properties>
        <log4j.version>2.5</log4j.version>
        <hadoop.version>2.7.2</hadoop.version>
        <scopeType>provided</scopeType>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.ikeguang</groupId>
            <artifactId>common</artifactId>
            <version>1.0-SNAPSHOT</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>{hadoop.version}</version>
            <scope>{scopeType}</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>{hadoop.version}</version>
            <scope>{scopeType}</scope>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-mapreduce-client-core -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-core</artifactId>
            <version>{hadoop.version}</version>
            <scope>{scopeType}</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-common</artifactId>
            <version>{hadoop.version}</version>
            <scope>{scopeType}</scope>
        </dependency>
    </dependencies>

package org.ikeguang.hadoop.mapreduce.wordcount;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

/**
 * Created by keguang on 2019-12-07.
 */
public class WordCountMapper extends Mapper<LongWritable, Text, Text, IntWritable> {

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String[] words = value.toString().split(" ");
        for(String word : words){
            context.write(new Text(word), new IntWritable(1));
        }
    }
}

package org.ikeguang.hadoop.mapreduce.wordcount;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

/**
 * Created by keguang on 2019-12-07.
 */
public class WordCountReducer extends Reducer<Text, IntWritable, Text, IntWritable> {

    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        int sum = 0;
        for(IntWritable val : values){
            sum = sum + val.get();
        }

        context.write(key, new IntWritable(sum));
    }
}

package org.ikeguang.hadoop.mapreduce.wordcount;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.CombineTextInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.ikeguang.hadoop.util.HdfsUtil;

/**
 * Created by keguang on 2019-12-07.
 */
public class WordCountDriver extends Configured implements Tool{

    public static void main(String[] args) throws Exception {
        int ec = ToolRunner.run(new Configuration(),new WordCountDriver(),args);
        System.exit(ec);
    }

    @Override
    public int run(String[] args) throws Exception {
        Configuration conf = new Configuration();
        Job job = Job.getInstance();
        job.setJobName("wordcount");

        job.setJarByClass(WordCountDriver.class);
        job.setMapperClass(WordCountMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        job.setReducerClass(WordCountReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        // 輸入輸出路徑
        String inpath = args[0];
        String output_path = args[1];

        FileInputFormat.addInputPath(job, new Path(inpath));

        if(HdfsUtil.existsFiles(conf,output_path)){
            HdfsUtil.deleteFolder(conf,output_path);
        }

        // 輸入路徑可以遞歸
        FileInputFormat.setInputDirRecursive(job,true);

        // 輸入數(shù)據(jù)小文件合并
        job.setInputFormatClass(CombineTextInputFormat.class);

        // 一個map最少處理128M文件
        CombineTextInputFormat.setMinInputSplitSize(job,134217728);
        // 最多處理256M文件
        CombineTextInputFormat.setMaxInputSplitSize(job,new Long(268435456));

        // job.setNumReduceTasks(10);

        // 輸出路徑
        FileOutputFormat.setOutputPath(job,new Path(output_path));

        return job.waitForCompletion(true)?0:1;
    }
}

hadoop jar hadoop-1.0-SNAPSHOT.jar org.ikeguang.hadoop.mapreduce.wordcount.WordCountDriver /data/wordcount/input /data/wordcount/output

select word, count(1) from table group by word;