Example Analysis of NLineInputFormat

The editor will share with you the example analysis of NLineInputFormat. I hope you will get something after reading this article. Let's discuss it together.

Package com.test;import java.io.IOException;import java.util.Iterator;import java.util.StringTokenizer;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.conf.Configured;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer The data processed by import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.input.NLineInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;import org.apache.hadoop.util.Tool;import org.apache.hadoop.util.ToolRunner;/** * TextInputFormat comes from an InputSplit. InputSplit is divided according to block size. * because each record is long and short, the number of records processed by each map task varies * NLineInputFormat determines that the number of records processed by each map is the same * / public class WordCountNL extends Configured implements Tool {public static class Map extends Mapper {public void map (LongWritable key, Text value, Context context) throws IOException, InterruptedException {String line = value.toString (); StringTokenizer st = new StringTokenizer (line) While (st.hasMoreElements ()) {context.write (new Text (st.nextElement (). ToString ()), new IntWritable (1));} public static class Combiner extends Reducer {public void reduce (Text key, Iterable values, Context context) throws IOException, InterruptedException {int count = 0; Iterator it = values.iterator (); while (it.hasNext ()) {count = count + it.next (). Get ();} context.write (key, new IntWritable (count)) }} public static class Reduce extends Reducer {public void reduce (Text key, Iterable values, Context context) throws IOException, InterruptedException {int count = 0; Iterator it = values.iterator (); while (it.hasNext ()) {count = count + it.next (). Get ();} context.write (key, new IntWritable (count));} public int run (String [] args) throws IOException, InterruptedException, ClassNotFoundException {Configuration conf = this.getConf () / / set how many rows of data each map can handle / / conf.set ("mapreduce.input.lineinputformat.linespermap", "1"); conf.set (NLineInputFormat.LINES_PER_MAP, "1"); Job job = new Job (conf); job.setJobName (WordCountNL.class.getSimpleName ()); job.setJarByClass (WordCountNL.class); FileInputFormat.addInputPath (job, new Path (args [0])); FileOutputFormat.setOutputPath (job, new Path (args [1]) Job.setMapperClass (Map.class); job.setCombinerClass (Combiner.class); job.setReducerClass (Reduce.class); job.setInputFormatClass (NLineInputFormat.class); job.setOutputFormatClass (TextOutputFormat.class); job.setOutputKeyClass (Text.class); job.setOutputValueClass (IntWritable.class); job.waitForCompletion (true); return job.isSuccessful ()? 0V1;} public static void main (String [] args) throws Exception {int exit = ToolRunner.run (new WordCount (), args) System.exit (exit);}} after reading this article, I believe you have some understanding of "sample Analysis of NLineInputFormat". If you want to know more about it, you are welcome to follow the industry information channel. Thank you for reading!