diff --git a/bench/scripts/collection-strawman.sh b/bench/scripts/collection-strawman.sh
new file mode 100755
index 000000000000..54724bbbd8f2
--- /dev/null
+++ b/bench/scripts/collection-strawman.sh
@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+find library/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench/jmh:run" {} + | sbt
diff --git a/bench/scripts/compiler.sh b/bench/scripts/compiler.sh
new file mode 100755
index 000000000000..26c15d9ff2a9
--- /dev/null
+++ b/bench/scripts/compiler.sh
@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+find collection-strawman/src/main/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench/jmh:run" {} + | sbt
diff --git a/bench/scripts/library.sh b/bench/scripts/library.sh
new file mode 100755
index 000000000000..26c15d9ff2a9
--- /dev/null
+++ b/bench/scripts/library.sh
@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+find collection-strawman/src/main/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench/jmh:run" {} + | sbt
diff --git a/bench/src/main/scala/Benchmarks.scala b/bench/src/main/scala/Benchmarks.scala
index a29739a93e6e..dd5cfe4df6ae 100644
--- a/bench/src/main/scala/Benchmarks.scala
+++ b/bench/src/main/scala/Benchmarks.scala
@@ -18,19 +18,26 @@ object Bench {
   val COMPILE_OPTS_FILE = "compile.txt"
 
   def main(args: Array[String]): Unit = {
-    storeCompileOptions(args)
+    if (args.isEmpty) {
+      println("Missing <args>")
+      return
+    }
 
-    val libs = System.getenv("BOOTSTRAP_APPEND")
+    val args2 = args.map { arg =>
+      if ((arg.endsWith(".scala") || arg.endsWith(".java")) && arg.head != '/') "../" + arg
+      else arg
+    }
+    storeCompileOptions(args2)
+
+    val libs = System.getProperty("BENCH_CLASS_PATH")
 
     val opts = new OptionsBuilder()
-               .jvmArgsPrepend(s"-Xbootclasspath/a:$libs")
+               .jvmArgsPrepend("-Xbootclasspath/a:" + libs + ":")
                .mode(Mode.AverageTime)
-               .timeUnit(TimeUnit.MICROSECONDS)
-               .forks(5)
-               .warmupIterations(5)
-               .measurementIterations(10)
-               .resultFormat(ResultFormatType.CSV)
-               .result("result.csv")
+               .timeUnit(TimeUnit.MILLISECONDS)
+               .forks(1)
+               .warmupIterations(12)
+               .measurementIterations(20)
                .build
 
     val runner = new Runner(opts) // full access to all JMH features, you can also provide a custom output Format here
diff --git a/bench/templates/launch.mustache b/bench/templates/launch.mustache
deleted file mode 100644
index ceae4fb01488..000000000000
--- a/bench/templates/launch.mustache
+++ /dev/null
@@ -1,147 +0,0 @@
-#!/bin/sh
-#/*--------------------------------------------------------------------------
-# *  Copyright 2012 Taro L. Saito
-# *
-# *  Licensed under the Apache License, Version 2.0 (the "License");
-# *  you may not use this file except in compliance with the License.
-# *  You may obtain a copy of the License at
-# *
-# *     http://www.apache.org/licenses/LICENSE-2.0
-# *
-# *  Unless required by applicable law or agreed to in writing, software
-# *  distributed under the License is distributed on an "AS IS" BASIS,
-# *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# *  See the License for the specific language governing permissions and
-# *  limitations under the License.
-# *--------------------------------------------------------------------------*/
-
-if [ -z "$PROG_HOME" ] ; then
-  ## resolve links - $0 may be a link to PROG_HOME
-  PRG="$0"
-
-  # need this for relative symlinks
-  while [ -h "$PRG" ] ; do
-    ls=`ls -ld "$PRG"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-      PRG="$link"
-    else
-      PRG="`dirname "$PRG"`/$link"
-    fi
-  done
-
-  saveddir=`pwd`
-
-  PROG_HOME=`dirname "$PRG"`/..
-
-  # make it fully qualified
-  PROG_HOME=`cd "$PROG_HOME" && pwd`
-
-  cd "$saveddir"
-fi
-
-
-cygwin=false
-mingw=false
-darwin=false
-case "`uname`" in
-  CYGWIN*) cygwin=true
-          ;;
-  MINGW*) mingw=true
-          ;;
-  Darwin*) darwin=true
-           if [ -z "$JAVA_VERSION" ] ; then
-             JAVA_VERSION="CurrentJDK"
-           else
-            echo "Using Java version: $JAVA_VERSION" 1>&2
-           fi
-           if [ -z "$JAVA_HOME" ] ; then
-             JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/${JAVA_VERSION}/Home
-           fi
-           JAVACMD="`which java`"
-           ;;
-esac
-
-# Resolve JAVA_HOME from javac command path
-if [ -z "$JAVA_HOME" ]; then
-  javaExecutable="`which javac`"
-  if [ -n "$javaExecutable" -a -f "$javaExecutable" -a ! "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then
-    # readlink(1) is not available as standard on Solaris 10.
-    readLink=`which readlink`
-    if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then
-      javaExecutable="`readlink -f \"$javaExecutable\"`"
-      javaHome="`dirname \"$javaExecutable\"`"
-      javaHome=`expr "$javaHome" : '\(.*\)/bin'`
-      JAVA_HOME="$javaHome"
-      export JAVA_HOME
-    fi
-  fi
-fi
-
-
-if [ -z "$JAVACMD" ] ; then
-  if [ -n "$JAVA_HOME"  ] ; then
-    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
-      # IBM's JDK on AIX uses strange locations for the executables
-      JAVACMD="$JAVA_HOME/jre/sh/java"
-    else
-      JAVACMD="$JAVA_HOME/bin/java"
-    fi
-  else
-    JAVACMD="`which java`"
-  fi
-fi
-
-if [ ! -x "$JAVACMD" ] ; then
-  echo "Error: JAVA_HOME is not defined correctly."
-  echo "  We cannot execute $JAVACMD"
-  exit 1
-fi
-
-if [ -z "$JAVA_HOME" ] ; then
-  echo "Warning: JAVA_HOME environment variable is not set."
-fi
-
-CLASSPATH_SUFFIX=""
-# Path separator used in EXTRA_CLASSPATH
-PSEP=":"
-
-# For Cygwin, switch paths to Windows-mixed format before running java
-if $cygwin; then
-  [ -n "$PROG_HOME" ] &&
-    PROG_HOME=`cygpath -am "$PROG_HOME"`
-  [ -n "$JAVA_HOME" ] &&
-    JAVA_HOME=`cygpath -am "$JAVA_HOME"`
-  CLASSPATH_SUFFIX=";"
-  PSEP=";"
-fi
-
-# For Migwn, ensure paths are in UNIX format before anything is touched
-if $mingw ; then
-  [ -n "$PROG_HOME" ] &&
-    PROG_HOME="`(cd "$PROG_HOME"; pwd -W | sed 's|/|\\\\|g')`"
-  [ -n "$JAVA_HOME" ] &&
-    JAVA_HOME="`(cd "$JAVA_HOME"; pwd -W | sed 's|/|\\\\|g')`"
-  CLASSPATH_SUFFIX=";"
-  PSEP=";"
-fi
-
-
-PROG_NAME={{{PROG_NAME}}}
-PROG_VERSION={{{PROG_VERSION}}}
-PROG_REVISION={{{PROG_REVISION}}}
-
-# add default libraries for compilation
-export BOOTSTRAP_APPEND="{{{EXPANDED_CLASSPATH}}}${CLASSPATH_SUFFIX}"
-
-eval exec "\"$JAVACMD\"" \
-     {{{JVM_OPTS}}} \
-     ${JAVA_OPTS} \
-     {{^EXPANDED_CLASSPATH}}
-     -cp "'{{{EXTRA_CLASSPATH}}}${PROG_HOME}/lib/*${CLASSPATH_SUFFIX}'" \
-     {{/EXPANDED_CLASSPATH}}
-     {{#EXPANDED_CLASSPATH}}
-     -cp "'{{{EXTRA_CLASSPATH}}}{{{EXPANDED_CLASSPATH}}}${CLASSPATH_SUFFIX}'" \
-     {{/EXPANDED_CLASSPATH}}
-     {{{MAIN_CLASS}}} \"\$@\"
-exit $?
diff --git a/project/Build.scala b/project/Build.scala
index b0f15229c870..ff255cfe2584 100644
--- a/project/Build.scala
+++ b/project/Build.scala
@@ -828,17 +828,10 @@ object Build {
     dependsOn(`dotty-compiler`).
     settings(commonNonBootstrappedSettings).
     settings(
-      mainClass in (Jmh, run) := Some("dotty.tools.benchmarks.Bench")   // custom main for jmh:run
+      mainClass in (Jmh, run) := Some("dotty.tools.benchmarks.Bench"), // custom main for jmh:run
+      javaOptions += "-DBENCH_CLASS_PATH=" + Attributed.data((fullClasspath in Compile).value).mkString("", ":", "")
     ).
-    enablePlugins(JmhPlugin).
-    settings(packSettings).
-    settings(
-      publishArtifact := false,
-      packMain := Map("bench" -> "dotty.tools.benchmarks.Bench"),
-      packGenerateWindowsBatFile := false,
-      packExpandedClasspath := true,
-      packBashTemplate := baseDirectory.value + "/templates/launch.mustache"
-    )
+    enablePlugins(JmhPlugin)
 
   // Depend on dotty-library so that sbt projects using dotty automatically
   // depend on the dotty-library