[hive] java.lang.OutOfMemoryError: Java heap space 오류 수정

하이브 tez 엔진 처리중 발생하는 

java.lang.OutOfMemoryError: Java heap space

오류는 다음과 같이 설정하여 수정할 수 있다. 

이번 경우는 OpenCSVSerde 에서 발생하였는데 다음과 같이 container size 를 수정하여 처리하였다. 

set hive.tez.container.size=4096;

set hive.tez.java.opts=-Xmx3280m;

], TaskAttempt 3 failed, info=[Error: Error while running task ( failure ) : attempt_1513225774189_104839_1_00_000002_3:java.lang.RuntimeException: java.lang.OutOfMemoryError: Java heap space

at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:181)

at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:160)

at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370)

at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)

at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)

at java.security.AccessController.doPrivileged(Native Method)

at javax.security.auth.Subject.doAs(Subject.java:422)

at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)

at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)

at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)

at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)

at java.util.concurrent.FutureTask.run(FutureTask.java:266)

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)

at java.lang.Thread.run(Thread.java:745)

Caused by: java.lang.OutOfMemoryError: Java heap space

at java.util.Arrays.copyOfRange(Arrays.java:3664)

at java.lang.StringBuffer.toString(StringBuffer.java:669)

at java.io.BufferedReader.readLine(BufferedReader.java:327)

at java.io.BufferedReader.readLine(BufferedReader.java:389)

at au.com.bytecode.opencsv.CSVReader.getNextLine(CSVReader.java:266)

at au.com.bytecode.opencsv.CSVReader.readNext(CSVReader.java:233)

at org.apache.hadoop.hive.serde2.OpenCSVSerde.deserialize(OpenCSVSerde.java:152)

at org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.readRow(MapOperator.java:128)

at org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.access$200(MapOperator.java:92)

at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:488)

at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.processRow(MapRecordSource.java:86)

at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:70)

at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.run(MapRecordProcessor.java:360)

at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:172)