Commit 36ebd63b authored by Arthur Bit-Monnot's avatar Arthur Bit-Monnot

Added analysis module for RCLL.

parent 10ae2ea7
......@@ -14,7 +14,7 @@ object RCLL extends App {
val TIMEOUT = 60.seconds
dahu.utils.debug.LOG_LEVEL = 2
for(game <- 1 to 30; c <- 0 to 1; r <- 1 to 3; spec <- Seq(true)) {
for(game <- 1 to 20; c <- 0 to 1; r <- 1 to 3; spec <- Seq(false, true)) {
val pbName = f"$game%03d-R$r-C$c"
val domain = new File("out/rcll.dom.pddl")
......@@ -44,12 +44,12 @@ object RCLL extends App {
val startTime = System.currentTimeMillis().millis
val result = solve(domain, problem, spec, a => maxActions(a.name))
val duration = (System.currentTimeMillis().millis - startTime).toMillis.toFloat / 1000
val duration = RCLLAnalysis.format(System.currentTimeMillis().millis - startTime)
val status = result match {
case Some(plan) => "OK"
case None => "__"
}
println(pbName + (if(spec) " spec " else " gen ") + f" \t$status \t$duration%02.3f")
println(pbName + (if(spec) "\tspec" else "\tgen") + s"\t$status\t$duration")
}
......
package dahu.planning.pddl.planner
import ammonite.ops._
import scala.collection.mutable
import scala.concurrent.duration._
case class Aggregate(numSolved: Int, meanRuntime: Duration) {
def combine(o: Aggregate): Aggregate = o match {
case Aggregate(ns2, mr2) =>
if(numSolved + ns2 == 0)
Aggregate.empty
else
Aggregate(numSolved + ns2, (meanRuntime * numSolved + mr2 * ns2) / (numSolved + ns2))
}
override def toString: String =
numSolved.toString + " " + "%.2f".formatLocal(java.util.Locale.US,
meanRuntime.toMillis.toFloat / 1000)
}
object Aggregate {
def empty: Aggregate = Aggregate(0, Duration.Zero)
}
case class Run(problemFlags: Seq[String], planner: String, runtime: Option[FiniteDuration]) {
val runtimeSecs = runtime match {
case Some(dur) => RCLLAnalysis.format(dur)
case None => "--"
}
val fullFlags = problemFlags :+ planner
def agg: Aggregate = runtime match {
case Some(dur) => Aggregate(1, dur)
case None => Aggregate.empty
}
override def toString: String = s"${problemFlags.mkString("-")} $planner $runtimeSecs"
}
object RCLLAnalysis extends App {
def format(d: Duration): String =
"%.2f".formatLocal(java.util.Locale.US, d.toMillis.toFloat / 1000)
def readAll(readers: (Path, String => Run)*): Seq[Run] =
readers.flatMap { case (p, f) => readResults(p, f) }
def readResults(path: Path, lineParser: String => Run): Seq[Run] = {
println(s"Reading $path")
read.lines(path).map(lineParser)
}
val formatLine: String => String = {
case "spec" => raw"\lcpspec"
case "gen" => raw"\lcpgen"
case "M1-o0" => raw"\omtsat"
case "M1-o1" => raw"\omtopt"
case "M2-o0" => raw"\macrosat"
case "M2-o1" => raw"\macroopt"
case "M-o" => raw"\optic"
}
val linePriority: String => Int = {
case "M1-o0" => 1
case "M1-o1" => 2
case "M2-o0" => 3
case "M2-o1" => 4
case "gen" => 6
case "spec" => 7
case "M-o" => 10
}
val leoRegex = raw"(\d{3})-(C\d)-(R\d)[\t ]+([^\t ]*)[\t ]+(.{2})[\t ]+(.*)".r
val lcpRegex = raw"(\d{3})-(R\d)-(C\d)[\t ]+([^\t ]*)[\t ]+(.{2})[\t ]+(\d+,\d+)".r
val timeRegex = raw"([0-9]*[\.])?([0-9]+)s?".r
def time(s: String): FiniteDuration = s match {
case timeRegex(t1, t2) =>
(t1 + t2).toFloat.seconds
}
val lineReader: String => Run = {
case leoRegex(game, c, r, planner, "OK", runtime) =>
Run(Seq(game, r, c), planner, Some(time(runtime)))
case leoRegex(g, c, r, planner, "__", _) =>
Run(Seq(g, r, c), planner, None)
case lcpRegex(g, r, c, planner, "OK", dur) =>
Run(Seq(g, r, c), planner, Some(dur.replace(',', '.').toFloat.seconds))
case lcpRegex(g, r, c, planner, "__", dur) =>
Run(Seq(g, r, c), planner, None)
}
val opticLineReader: String => Run = {
case leoRegex(game, c, r, planner, "OK", runtime) =>
val t = time(runtime)
if(t < 0.1.seconds)
Run(Seq(game, r, c), planner, None)
else
Run(Seq(game, r, c), planner, Some(time(runtime)))
case leoRegex(g, c, r, planner, "__", _) =>
Run(Seq(g, r, c), planner, None)
}
val runs =
readAll(
pwd / "MACRO.txt" -> lineReader,
pwd / "PURE-SAT.txt" -> lineReader,
pwd / "PURE-OPT.txt" -> lineReader,
pwd / "LCP.txt" -> lineReader,
pwd / "OPTIC.txt" -> opticLineReader
)
val planners = runs.map(_.planner).toSet
val problemsByPlanner = runs.groupBy(_.planner).mapValues(rs => rs.map(_.problemFlags).toSet)
val commonProblems =
runs.map(_.problemFlags).toSet.filter(pb => problemsByPlanner.values.forall(_.contains(pb)))
val sharedRuns = runs.filter(r => commonProblems.contains(r.problemFlags))
sharedRuns.foreach(println)
val splitOnFlags =
Seq("C0", "C1").map(c => (c, sharedRuns.filter(_.problemFlags.contains(c)))).toMap
println(splitOnFlags)
def aggregate(runs: Seq[Run], lineID: Int, colID: Int): Map[(String, String), Aggregate] = {
val res = mutable.Map[(String, String), Aggregate]()
def line(r: Run): String = r.fullFlags(lineID)
def col(r: Run): String = r.fullFlags(colID)
def curAgg(r: Run): Aggregate = res.getOrElse((line(r), col(r)), Aggregate.empty)
runs.foreach { r =>
res.update((line(r), col(r)), curAgg(r).combine(r.agg))
}
res.toMap
}
for((comp, runs) <- splitOnFlags) {
println(comp)
val agg = aggregate(runs, 3, 1)
val lines = agg.keys.map(_._1).toSeq.sortBy(linePriority)
val cols = agg.keys.map(_._2).toSeq.sorted
val bestSolved: Map[String, Int] =
cols
.map(c => c -> lines.map(l => agg((l, c)).numSolved).foldLeft(0)((a, b) => math.max(a, b)))
.toMap
val bestRuntime: Map[String, Int] =
cols
.map(
c =>
c -> lines
.filter(l => agg((l, c)).numSolved != 0)
.map(l => agg((l, c)).meanRuntime.toMillis.toInt)
.foldLeft(Int.MaxValue)((a, b) => math.min(a, b)))
.toMap
for(l <- lines) {
print(formatLine(l))
for(c <- cols) {
print(" & ")
val a = agg((l, c))
val X =
if(a.numSolved == bestSolved(c))
s"\\textbf{${a.numSolved}}"
else
a.numSolved.toString
val Y =
if(a.numSolved == 0)
"--"
else if(a.meanRuntime.toMillis.toInt == bestRuntime(c))
s"\\textbf{${format(a.meanRuntime)}}"
else
format(a.meanRuntime)
print(s"$X & $Y")
}
println(raw" \\\hline")
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment