MapReduce Function with JSON Files and JSONParser - json

Question

i have some problems during writing my mapreduce funtions.
I want to solve the following problem:
I have a JSON file with 1mio JSONObject like this:
{"_id":3951,"title":"Two Family House (2000)","genres":["Drama"],"ratings":[{"userId":173,"rating":5},{"userId":195,"rating":5},{"userId":411,"rating":4},{"userId":593,"rating":2},{"userId":629,"rating":3},{"userId":830,"rating":3},{"userId":838,"rating":5},{"userId":850,"rating":4},{"userId":856,"rating":4},{"userId":862,"rating":5},{"userId":889,"rating":1},{"userId":928,"rating":5},{"userId":986,"rating":4},{"userId":1001,"rating":5},{"userId":1069,"rating":3},{"userId":1168,"rating":3},{"userId":1173,"rating":2},{"userId":1242,"rating":3},{"userId":1266,"rating":5},{"userId":1331,"rating":5},{"userId":1417,"rating":5},{"userId":1470,"rating":4},{"userId":1474,"rating":5},{"userId":1615,"rating":3},{"userId":1625,"rating":4},{"userId":1733,"rating":4},{"userId":1799,"rating":4},{"userId":1865,"rating":5},{"userId":1877,"rating":5},{"userId":1897,"rating":5},{"userId":1946,"rating":4},{"userId":2031,"rating":4},{"userId":2129,"rating":2},{"userId":2353,"rating":4},{"userId":2986,"rating":4},{"userId":3940,"rating":4},{"userId":3985,"rating":3},{"userId":4025,"rating":5},{"userId":4727,"rating":3},{"userId":5333,"rating":3}]}
and more....
One JSON Object is a Movie, which contains a array ratings. I want to count all ratings in the JSON File.
I created a Maven Proct in IntelliJ with the dependencys for Hadoop and JSON Parser. My MapReduce Class is this:
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import java.io.IOException;
import java.util.Iterator;
public class RatingCounter {
public static class RatingMapper extends Mapper<JSONObject, Text, Text, Text>{
private Text id = new Text();
private Text ratingAnzahl = new Text();
public void map(LongWritable key, Text value, OutputCollector<Text, Text> output, Reporter reporter) throws IOException{
JSONParser parser = new JSONParser();
try {
Object obj = parser.parse(value.toString());
JSONObject jsonObject = (JSONObject) obj;
String movieId = (String) jsonObject.get("_id");
int count = 0;
// loop array
JSONArray ratings = (JSONArray) jsonObject.get("ratings");
Iterator<String> iterator = ratings.iterator();
while (iterator.hasNext()) {
count++;
}
} catch (ParseException e) {
e.printStackTrace();
}
}
}
public static class RatingReducer extends Reducer<Text, Text, Text, Text> {
public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
Text resultValue = new Text();
int allRatings = 0;
while (values.hasNext()){
allRatings += Integer.parseInt(values.toString());
}
resultValue.set(""+allRatings);
context.write(key, resultValue);
}
}
public static void main (String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = new Job(conf, "ratings count");
job.setJarByClass(RatingCounter.class);
job.setMapperClass(RatingMapper.class);
job.setReducerClass(RatingReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
I have no idea, how I can write the functions in Mapper and Reducer. Can someone help me pls?

Answer 1

I've made a few changes to your mapper and reducer.
First, for your mapper, you are not writing the output anywhere and your syntax while extending the Mapper class is also wrong(arguably). The first input to any mapper is a LongWritable (or Object type) offset of line. You can notice the changes below
public static class RatingMapper extends Mapper<LongWritable, Text, Text, IntWritable>{
public void map(LongWritable key, Text value, Context context) throws IOException, ParseException{
JSONParser parser = new JSONParser();
Object obj = parser.parse(value.toString());
JSONObject jsonObject = (JSONObject) obj;
String movieId = (String) jsonObject.get("_id");
JSONArray ratings = (JSONArray) jsonObject.get("ratings");
context.write(new Text(movieId), new IntWritable(ratings.size()) );
}
}
Notice here, the output of map is written using context.write
Now, coming onto your Reducer some things will change because of the changes I made in the mapper. Also, since your Number of Ratings will always be an integer, you don't need to convert it to Text, use parseInt and then convert to Text again.
public static class RatingReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
int allRatings = 0;
while (values.hasNext()){
allRatings += value.get();
}
context.write(key, new IntWritable(resultValue));
}
}