我在做一个hadoop项目。我的reduce阶段内存非常昂贵。我正在使用一个hashmap，但是我得到了 Error: Java Heap space 因为在reduce中我构建了一个巨大的hashmap（数据集是32gb）。解决方案可以是内存中的hashmap和磁盘回退，mapdb似乎适合我的需要。但我不确定它的用法。diskmap对于每个reduce任务是唯一的，inmemoryMap对于每个reduce“key”是唯一的。即使我设定 expireMaxSize(3) 对于测试，我不确定何时使用ondiskMap以及逻辑是否正确。同样，为了测试，我用20个假条目填充hashmap。基本上，为了避免堆溢出，我需要控制内存Map的增长。

public class TestReducer extends Reducer<LongWritable, BytesWritable, String, IntWritable> {

private int id;
DB dbDisk;
protected void setup(Context context) throws IOException, InterruptedException {
    id = context.getTaskAttemptID().getTaskID().getId();
    File diskmap = new File("tmp/diskmap"+id);
    diskmap.delete();
    dbDisk = DBMaker
                .fileDB("tmp/diskmap"+id)
                .make();
}

@Override
protected void reduce(LongWritable key, Iterable<BytesWritable> values, Context context)
        throws IOException, InterruptedException {

    DB dbMemory = DBMaker
            .memoryDB()
            .make();

    HTreeMap<Long,Integer> onDisk = dbDisk
            .hashMap("onDisk")
            .keySerializer(Serializer.LONG)
            .valueSerializer(Serializer.INTEGER)
            .createOrOpen();
    // fast in-memory collection with limited size
    HTreeMap<Long,Integer> inMemory = dbMemory
            .hashMap("inMemory")
            .expireMaxSize(3)
            .keySerializer(Serializer.LONG)
            .valueSerializer(Serializer.INTEGER)
            //this registers overflow to `onDisk`
            .expireOverflow(onDisk)
            .createOrOpen();

    for(int k=0;k<20;k++){
        inMemory.put((long)k,k*2);
    }
Set set = inMemory.entrySet();
    Iterator it = set.iterator();
    while(it.hasNext()) {
      Map.Entry<Long,Integer> entry = (Map.Entry<Long,Integer>)it.next();
      System.out.print("Key is: "+entry.getKey() + " & ");
      System.out.println("Value is: "+entry.getValue());
    }

}
protected void cleanup(Context context) throws IOException,InterruptedException {
    dbDisk.close();
}

}