Scatter task with ~2000 jobs runs out-of-memory due to database flush
open | Created 2018-03-09 | Last updated 2019-02-21| Posted by magicDGS | See in Github


Needs Triage User Requested Improvement


Running locally a scatter job with 2102 jobs causes an out-of-memory error after finishing job 1427 (reproduced twice) . The command that I run was using the wrapper script from brew:

JAVA_OPTS="-Dconfig.file=local.conf" cromwell run --inputs inputs.json --metadata-output metadata-output.json workflow.wdl

Where the local configuration looks like this:

include required(classpath("application"))

## keep always the workflow logs
workflow-options.workflow-log-temporary: false
workflow-options.workflow-log-dir: "/Volumes/Temp/E43CEE02/data/freqs/haf/base-all/w100000_2.0x/workflow-logs"

# Allows re-use of existing results for jobs you've already run
call-caching.enabled: true

backend.providers.Local.config {
    ## limit the number of jobs
    concurrent-job-limit = 10
    # set the root directory to the run
    root = "/Volumes/Temp/E43CEE02/data/freqs/haf/base-all/w100000_2.0x/execution"
    filesystems.local {
        ## do not allow copy (too huge files)
        ## prefer hard-links, to don't remove data and kept analysis intact
        localization: ["hard-link", "soft-link"]
        caching.duplication-strategy: ["hard-link", "soft-link"]
    }
    # custom submit-docker to workaround detached container due to timeout in the virtual machine
    # first, we do not remove the container until it really finishes (no --rm flag)
    # if the docker run command fails, then it runs docker wait to wait until it finishes and store the return code
    # if the docker run command fails, then it runs docker wait to return the real exit code even if detached
    # once it finishes, removes the docker container with docker rm
    # finally, returns the "real return code" stored
    submit-docker = """
        docker run \
          --cidfile ${docker_cid} \
          -i \
          ${"--user " + docker_user} \
          --entrypoint /bin/bash \
          -v ${cwd}:${docker_cwd} \
          ${docker} ${script}
        rc=$(docker wait `cat ${docker_cid}`)
        docker rm `cat ${docker_cid}`
        exit $rc
    """
}

The log shows the following stack-trace:

[2018-03-09 15:31:16,47] [error] Failed to properly flush metadata to database
java.sql.SQLException: java.lang.OutOfMemoryError: Java heap space
	at org.hsqldb.jdbc.JDBCUtil.sqlException(Unknown Source)
	at org.hsqldb.jdbc.JDBCUtil.sqlException(Unknown Source)
	at org.hsqldb.jdbc.JDBCPreparedStatement.addBatch(Unknown Source)
	at com.zaxxer.hikari.pool.HikariProxyPreparedStatement.addBatch(HikariProxyPreparedStatement.java)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction.$anonfun$run$15(JdbcActionComponent.scala:531)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction.$anonfun$run$15$adapted(JdbcActionComponent.scala:529)
	at scala.collection.Iterator.foreach(Iterator.scala:929)
	at scala.collection.Iterator.foreach$(Iterator.scala:929)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1417)
	at scala.collection.IterableLike.foreach(IterableLike.scala:71)
	at scala.collection.IterableLike.foreach$(IterableLike.scala:70)
	at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction.$anonfun$run$14(JdbcActionComponent.scala:529)
	at slick.jdbc.JdbcBackend$SessionDef.withPreparedStatement(JdbcBackend.scala:372)
	at slick.jdbc.JdbcBackend$SessionDef.withPreparedStatement$(JdbcBackend.scala:367)
	at slick.jdbc.JdbcBackend$BaseSession.withPreparedStatement(JdbcBackend.scala:434)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl.preparedInsert(JdbcActionComponent.scala:502)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction.run(JdbcActionComponent.scala:527)
	at slick.jdbc.JdbcActionComponent$SimpleJdbcProfileAction.run(JdbcActionComponent.scala:31)
	at slick.jdbc.JdbcActionComponent$SimpleJdbcProfileAction.run(JdbcActionComponent.scala:28)
	at slick.dbio.DBIOAction$$anon$4.$anonfun$run$3(DBIOAction.scala:240)
	at scala.collection.Iterator.foreach(Iterator.scala:929)
	at scala.collection.Iterator.foreach$(Iterator.scala:929)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1417)
	at scala.collection.IterableLike.foreach(IterableLike.scala:71)
	at scala.collection.IterableLike.foreach$(IterableLike.scala:70)
	at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
	at slick.dbio.DBIOAction$$anon$4.run(DBIOAction.scala:240)
	at slick.dbio.DBIOAction$$anon$4.run(DBIOAction.scala:238)
	at slick.dbio.SynchronousDatabaseAction$FusedAndThenAction.$anonfun$run$4(DBIOAction.scala:534)
	at slick.dbio.SynchronousDatabaseAction$FusedAndThenAction.$anonfun$run$4$adapted(DBIOAction.scala:534)
	at scala.collection.Iterator.foreach(Iterator.scala:929)
	at scala.collection.Iterator.foreach$(Iterator.scala:929)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1417)
	at scala.collection.IterableLike.foreach(IterableLike.scala:71)
	at scala.collection.IterableLike.foreach$(IterableLike.scala:70)
	at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
	at slick.dbio.SynchronousDatabaseAction$FusedAndThenAction.run(DBIOAction.scala:534)
	at slick.dbio.SynchronousDatabaseAction$$anon$11.run(DBIOAction.scala:571)
	at slick.basic.BasicBackend$DatabaseDef$$anon$2.liftedTree1$1(BasicBackend.scala:240)
	at slick.basic.BasicBackend$DatabaseDef$$anon$2.run(BasicBackend.scala:240)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.hsqldb.HsqlException: java.lang.OutOfMemoryError: Java heap space
	at org.hsqldb.error.Error.error(Unknown Source)
	at org.hsqldb.SessionData.allocateLobForResult(Unknown Source)
	at org.hsqldb.Session.allocateResultLob(Unknown Source)
	at org.hsqldb.jdbc.JDBCPreparedStatement.performPreExecute(Unknown Source)
	... 42 common frames omitted
Caused by: java.lang.OutOfMemoryError: Java heap space
	at org.hsqldb.persist.LobStoreMem.setBlockBytes(Unknown Source)
	at org.hsqldb.persist.LobManager.setBytesISNormal(Unknown Source)
	at org.hsqldb.persist.LobManager.setBytesIS(Unknown Source)
	at org.hsqldb.persist.LobManager.setCharsForNewClob(Unknown Source)
	at org.hsqldb.SessionData.allocateLobForResult(Unknown Source)
	at org.hsqldb.Session.allocateResultLob(Unknown Source)
	at org.hsqldb.jdbc.JDBCPreparedStatement.performPreExecute(Unknown Source)
	at org.hsqldb.jdbc.JDBCPreparedStatement.addBatch(Unknown Source)
	at com.zaxxer.hikari.pool.HikariProxyPreparedStatement.addBatch(HikariProxyPreparedStatement.java)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction.$anonfun$run$15(JdbcActionComponent.scala:531)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction.$anonfun$run$15$adapted(JdbcActionComponent.scala:529)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction$$Lambda$1955/223664727.apply(Unknown Source)
	at scala.collection.Iterator.foreach(Iterator.scala:929)
	at scala.collection.Iterator.foreach$(Iterator.scala:929)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1417)
	at scala.collection.IterableLike.foreach(IterableLike.scala:71)
	at scala.collection.IterableLike.foreach$(IterableLike.scala:70)
	at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction.$anonfun$run$14(JdbcActionComponent.scala:529)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction$$Lambda$1954/1995472759.apply(Unknown Source)
	at slick.jdbc.JdbcBackend$SessionDef.withPreparedStatement(JdbcBackend.scala:372)
	at slick.jdbc.JdbcBackend$SessionDef.withPreparedStatement$(JdbcBackend.scala:367)
	at slick.jdbc.JdbcBackend$BaseSession.withPreparedStatement(JdbcBackend.scala:434)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl.preparedInsert(JdbcActionComponent.scala:502)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction.run(JdbcActionComponent.scala:527)

	at slick.jdbc.JdbcActionComponent$SimpleJdbcProfileAction.run(JdbcActionComponent.scala:31)
	at slick.jdbc.JdbcActionComponent$SimpleJdbcProfileAction.run(JdbcActionComponent.scala:28)
	at slick.dbio.DBIOAction$$anon$4.$anonfun$run$3(DBIOAction.scala:240)
	at slick.dbio.DBIOAction$$anon$4$$Lambda$1952/113291290.apply(Unknown Source)
	at scala.collection.Iterator.foreach(Iterator.scala:929)
	at scala.collection.Iterator.foreach$(Iterator.scala:929)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1417)
[2018-03-09 15:38:57,90] [warn] Couldn't find a suitable DSN, defaulting to a Noop one.
[2018-03-09 15:42:11,14] [info] Using noop to send events.
[2018-03-09 15:49:48,68] [warn] Localhost hostname lookup failed, keeping the value 'unavailable'
java.util.concurrent.TimeoutException: null
	at java.util.concurrent.FutureTask.get(FutureTask.java:205)
	at com.getsentry.raven.event.EventBuilder$HostnameCache.updateCache(EventBuilder.java:491)
	at com.getsentry.raven.event.EventBuilder$HostnameCache.getHostname(EventBuilder.java:477)
	at com.getsentry.raven.event.EventBuilder.autoSetMissingValues(EventBuilder.java:97)
	at com.getsentry.raven.event.EventBuilder.build(EventBuilder.java:410)
	at com.getsentry.raven.logback.SentryAppender.buildEvent(SentryAppender.java:324)
	at com.getsentry.raven.logback.SentryAppender.append(SentryAppender.java:230)
	at com.getsentry.raven.logback.SentryAppender.append(SentryAppender.java:37)
	at ch.qos.logback.core.AppenderBase.doAppend(AppenderBase.java:82)
	at ch.qos.logback.core.spi.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:51)
	at ch.qos.logback.classic.Logger.appendLoopOnAppenders(Logger.java:270)
	at ch.qos.logback.classic.Logger.callAppenders(Logger.java:257)
	at ch.qos.logback.classic.Logger.buildLoggingEventAndAppend(Logger.java:421)
	at ch.qos.logback.classic.Logger.filterAndLog_0_Or3Plus(Logger.java:383)
	at ch.qos.logback.classic.Logger.error(Logger.java:558)
	at akka.event.slf4j.Slf4jLogger$$anonfun$receive$1.$anonfun$applyOrElse$1(Slf4jLogger.scala:69)
	at akka.event.slf4j.Slf4jLogger.withMdc(Slf4jLogger.scala:100)
	at akka.event.slf4j.Slf4jLogger$$anonfun$receive$1.applyOrElse(Slf4jLogger.scala:65)
	at akka.actor.Actor.aroundReceive(Actor.scala:514)
	at akka.actor.Actor.aroundReceive$(Actor.scala:512)
	at akka.event.slf4j.Slf4jLogger.aroundReceive(Slf4jLogger.scala:54)
	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:527)
	at akka.actor.ActorCell.invoke(ActorCell.scala:496)
	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:257)
	at akka.dispatch.Mailbox.run(Mailbox.scala:224)
	at akka.dispatch.Mailbox.exec(Mailbox.scala:234)
	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
[2018-03-09 15:52:58,29] [error] Failed to properly flush metadata to database
java.sql.SQLException: java.lang.OutOfMemoryError: Java heap space
	at org.hsqldb.jdbc.JDBCUtil.sqlException(Unknown Source)
	at org.hsqldb.jdbc.JDBCUtil.sqlException(Unknown Source)
	at org.hsqldb.jdbc.JDBCPreparedStatement.addBatch(Unknown Source)
	at com.zaxxer.hikari.pool.HikariProxyPreparedStatement.addBatch(HikariProxyPreparedStatement.java)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction.$anonfun$run$15(JdbcActionComponent.scala:531)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction.$anonfun$run$15$adapted(JdbcActionComponent.scala:529)
	at scala.collection.Iterator.foreach(Iterator.scala:929)
	at scala.collection.Iterator.foreach$(Iterator.scala:929)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1417)
	at scala.collection.IterableLike.foreach(IterableLike.scala:71)
	at scala.collection.IterableLike.foreach$(IterableLike.scala:70)
	at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction.$anonfun$run$14(JdbcActionComponent.scala:529)
	at slick.jdbc.JdbcBackend$SessionDef.withPreparedStatement(JdbcBackend.scala:372)
	at slick.jdbc.JdbcBackend$SessionDef.withPreparedStatement$(JdbcBackend.scala:367)
	at slick.jdbc.JdbcBackend$BaseSession.withPreparedStatement(JdbcBackend.scala:434)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl.preparedInsert(JdbcActionComponent.scala:502)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction.run(JdbcActionComponent.scala:527)
	at slick.jdbc.JdbcActionComponent$SimpleJdbcProfileAction.run(JdbcActionComponent.scala:31)
	at slick.jdbc.JdbcActionComponent$SimpleJdbcProfileAction.run(JdbcActionComponent.scala:28)
	at slick.dbio.DBIOAction$$anon$4.$anonfun$run$3(DBIOAction.scala:240)
	at scala.collection.Iterator.foreach(Iterator.scala:929)
	at scala.collection.Iterator.foreach$(Iterator.scala:929)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1417)
	at scala.collection.IterableLike.foreach(IterableLike.scala:71)
	at scala.collection.IterableLike.foreach$(IterableLike.scala:70)
	at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
	at slick.dbio.DBIOAction$$anon$4.run(DBIOAction.scala:240)
	at slick.dbio.DBIOAction$$anon$4.run(DBIOAction.scala:238)
	at slick.dbio.SynchronousDatabaseAction$FusedAndThenAction.$anonfun$run$4(DBIOAction.scala:534)
	at slick.dbio.SynchronousDatabaseAction$FusedAndThenAction.$anonfun$run$4$adapted(DBIOAction.scala:534)
	at scala.collection.Iterator.foreach(Iterator.scala:929)
	at scala.collection.Iterator.foreach$(Iterator.scala:929)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1417)
	at scala.collection.IterableLike.foreach(IterableLike.scala:71)
	at scala.collection.IterableLike.foreach$(IterableLike.scala:70)
	at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
	at slick.dbio.SynchronousDatabaseAction$FusedAndThenAction.run(DBIOAction.scala:534)
	at slick.dbio.SynchronousDatabaseAction$$anon$11.run(DBIOAction.scala:571)
	at slick.basic.BasicBackend$DatabaseDef$$anon$2.liftedTree1$1(BasicBackend.scala:240)
	at slick.basic.BasicBackend$DatabaseDef$$anon$2.run(BasicBackend.scala:240)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.hsqldb.HsqlException: java.lang.OutOfMemoryError: Java heap space
	at org.hsqldb.error.Error.error(Unknown Source)
	at org.hsqldb.SessionData.allocateLobForResult(Unknown Source)
	at org.hsqldb.Session.allocateResultLob(Unknown Source)
	at org.hsqldb.jdbc.JDBCPreparedStatement.performPreExecute(Unknown Source)
	... 42 common frames omitted
Caused by: java.lang.OutOfMemoryError: Java heap space
	at org.hsqldb.persist.LobStoreMem.setBlockBytes(Unknown Source)
	at org.hsqldb.persist.LobManager.setBytesISNormal(Unknown Source)
	at org.hsqldb.persist.LobManager.setBytesIS(Unknown Source)
	at org.hsqldb.persist.LobManager.setCharsForNewClob(Unknown Source)
	at org.hsqldb.SessionData.allocateLobForResult(Unknown Source)
	at org.hsqldb.Session.allocateResultLob(Unknown Source)
	at org.hsqldb.jdbc.JDBCPreparedStatement.performPreExecute(Unknown Source)
	at org.hsqldb.jdbc.JDBCPreparedStatement.addBatch(Unknown Source)
	at com.zaxxer.hikari.pool.HikariProxyPreparedStatement.addBatch(HikariProxyPreparedStatement.java)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction.$anonfun$run$15(JdbcActionComponent.scala:531)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction.$anonfun$run$15$adapted(JdbcActionComponent.scala:529)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction$$Lambda$1955/223664727.apply(Unknown Source)
	at scala.collection.Iterator.foreach(Iterator.scala:929)
	at scala.collection.Iterator.foreach$(Iterator.scala:929)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1417)
	at scala.collection.IterableLike.foreach(IterableLike.scala:71)

	at scala.collection.IterableLike.foreach$(IterableLike.scala:70)
	at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction.$anonfun$run$14(JdbcActionComponent.scala:529)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction$$Lambda$1954/1995472759.apply(Unknown Source)
	at slick.jdbc.JdbcBackend$SessionDef.withPreparedStatement(JdbcBackend.scala:372)
	at slick.jdbc.JdbcBackend$SessionDef.withPreparedStatement$(JdbcBackend.scala:367)
	at slick.jdbc.JdbcBackend$BaseSession.withPreparedStatement(JdbcBackend.scala:434)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl.preparedInsert(JdbcActionComponent.scala:502)
	at slick.jdbc.JdbcActionComponent$InsertActionComposerImpl$MultiInsertAction.run(JdbcActionComponent.scala:527)
	at slick.jdbc.JdbcActionComponent$SimpleJdbcProfileAction.run(JdbcActionComponent.scala:31)
	at slick.jdbc.JdbcActionComponent$SimpleJdbcProfileAction.run(JdbcActionComponent.scala:28)
	at slick.dbio.DBIOAction$$anon$4.$anonfun$run$3(DBIOAction.scala:240)
	at slick.dbio.DBIOAction$$anon$4$$Lambda$1952/113291290.apply(Unknown Source)
	at scala.collection.Iterator.foreach(Iterator.scala:929)
	at scala.collection.Iterator.foreach$(Iterator.scala:929)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1417)
[2018-03-09 15:56:36,88] [warn] Localhost hostname lookup failed, keeping the value 'unavailable'
java.util.concurrent.TimeoutException: null
	at java.util.concurrent.FutureTask.get(FutureTask.java:205)
	at com.getsentry.raven.event.EventBuilder$HostnameCache.updateCache(EventBuilder.java:491)
	at com.getsentry.raven.event.EventBuilder$HostnameCache.getHostname(EventBuilder.java:477)
	at com.getsentry.raven.event.EventBuilder.autoSetMissingValues(EventBuilder.java:97)
	at com.getsentry.raven.event.EventBuilder.build(EventBuilder.java:410)
	at com.getsentry.raven.logback.SentryAppender.buildEvent(SentryAppender.java:324)
	at com.getsentry.raven.logback.SentryAppender.append(SentryAppender.java:230)
	at com.getsentry.raven.logback.SentryAppender.append(SentryAppender.java:37)
	at ch.qos.logback.core.AppenderBase.doAppend(AppenderBase.java:82)
	at ch.qos.logback.core.spi.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:51)
	at ch.qos.logback.classic.Logger.appendLoopOnAppenders(Logger.java:270)
	at ch.qos.logback.classic.Logger.callAppenders(Logger.java:257)
	at ch.qos.logback.classic.Logger.buildLoggingEventAndAppend(Logger.java:421)
	at ch.qos.logback.classic.Logger.filterAndLog_0_Or3Plus(Logger.java:383)
	at ch.qos.logback.classic.Logger.error(Logger.java:558)
	at akka.event.slf4j.Slf4jLogger$$anonfun$receive$1.$anonfun$applyOrElse$1(Slf4jLogger.scala:69)
	at akka.event.slf4j.Slf4jLogger.withMdc(Slf4jLogger.scala:100)
	at akka.event.slf4j.Slf4jLogger$$anonfun$receive$1.applyOrElse(Slf4jLogger.scala:65)
	at akka.actor.Actor.aroundReceive(Actor.scala:514)
	at akka.actor.Actor.aroundReceive$(Actor.scala:512)
	at akka.event.slf4j.Slf4jLogger.aroundReceive(Slf4jLogger.scala:54)
	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:527)
	at akka.actor.ActorCell.invoke(ActorCell.scala:496)
	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:257)
	at akka.dispatch.Mailbox.run(Mailbox.scala:224)
	at akka.dispatch.Mailbox.exec(Mailbox.scala:234)
	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

Return to top