MapReduce关于类型转换报错记录

精选原创

WHYBIGDATA 2023-03-11 07:56:27 博主文章分类：Hadoop ©著作权

文章标签 hadoop java MapReduce 大数据 Linux 文章分类 Hadoop 大数据 yyds干货盘点

©著作权归作者所有：来自51CTO博客作者WHYBIGDATA的原创作品，请联系作者获取转载授权，否则将追究法律责任

MapReduce关于类型转换报错记录

0. 写在前面

实验环境：Ubuntu Kylin16.04
Hadoop版本：2.7.2
IDE：Eclipse3.8

1. 程序代码

Mapper端

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public static class WordCountMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
  @Override
  protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
    String[] strs = value.toString().split(" ");
    for (String str : strs) {
      context.write(new Text(str), new IntWritable(1));
    }
  }
}

Reducer端

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public static class WordCountReduce extends Reducer<Text, IntWritable, Text, IntWritable> {
  
  @Override
  protected void reduce(Text key, Iterable<IntWritable> values, Context context)
      throws IOException, InterruptedException {

    int sum = 0;
    for (IntWritable val : values) {
      System.out.println("<" + key + "," + val + ">");
      sum += val.get();
    }
    context.write(key, new IntWritable(sum));
  }
}

Driver端

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class TxtCntDemo {
  
  public static void main(String[] args) throws Exception {
    args = new String[] { "/input", "/output"};

    Configuration conf = new Configuration();


    Job job = Job.getInstance(conf);/
    job.setJarByClass(TxtCntDemo.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setMapperClass(WordCountMap.class);
    job.setReducerClass(WordCountReduce.class);
    job.setPartitionerClass(MyPartitioner.class);
    job.setNumReduceTasks(4);
    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.waitForCompletion(true);

  }
}

「错误描述」

java.lang.Exception: java.lang.ClassCastException: org.apache.hadoop.io.Text cannot be cast to org.apache.hadoop.io.IntWritable

java.lang.Exception: java.lang.ClassCastException: org.apache.hadoop.io.LongWritable cannot be cast to org.apache.hadoop.io.IntWritable

mapper、reducer、driver分开成3个文件，报Text不可转换成IntWritable，还有LongWritable不能转换成IntWritable的错误

关于第二个错误：Mapper端执行时，key的默认输入是LongWritable类型，把LongWritable类型强行转换成Text类型自然就Error了。

java.lang.Exception: java.lang.ClassCastException: org.apache.hadoop.io.Text cannot be cast to org.apache.hadoop.io.IntWritable
  at org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:462)
  at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:522)
Caused by: java.lang.ClassCastException: org.apache.hadoop.io.Text cannot be cast to org.apache.hadoop.io.IntWritable
  at org.apache.hadoop.mapred.MapTask$NewOutputCollector.write(MapTask.java:715)
  at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89)
  at org.apache.hadoop.mapreduce.lib.map.WrappedMapper$Context.write(WrappedMapper.java:112)
  at cn.mr.WordCountMapper.map(WordCountMapper.java:15)
  at cn.mr.WordCountMapper.map(WordCountMapper.java:1)
  at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:146)
  at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:787)
  at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
  at org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:243)
  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
  at java.util.concurrent.FutureTask.run(FutureTask.java:266)
  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  at java.lang.Thread.run(Thread.java:748)