Store project/trace metadata in a single database (performance related)

Author: baffles

codepulse/src/main/scala/com/secdec/codepulse/data/trace/slick/SlickH2TraceDataProvider.scala

@@ -53,13 +53,24 @@ class SlickH2TraceDataProvider(folder: File, actorSystem: ActorSystem) extends T
 		}
 	}
 
+	private val masterData = {
+		val needsInit = !(folder / "master.h2.db").exists
+
+		val db = Database.forURL(s"jdbc:h2:file:${(folder / "master").getCanonicalPath};DB_CLOSE_DELAY=10", driver = "org.h2.Driver")
+		val data = new SlickMasterData(db, H2Driver)
+
+		if (needsInit) data.init
+
+		data
+	}
+
 	def getTrace(id: TraceId): TraceData = getTraceInternal(id)
 
 	private def getTraceInternal(id: TraceId, suppressInit: Boolean = false) = cache.getOrElseUpdate(id, {
 		val needsInit = !(folder / TraceFilename(id)).exists
 
 		val db = Database.forURL(s"jdbc:h2:file:${(folder / TraceFilename.getDbName(id)).getCanonicalPath};DB_CLOSE_DELAY=10", driver = "org.h2.Driver")
-		val data = new SlickTraceData(db, H2Driver, EncountersBufferSize, EncountersFlushInterval, actorSystem)
+		val data = new SlickTraceData(db, H2Driver, masterData.metadataMaster get id.num, EncountersBufferSize, EncountersFlushInterval, actorSystem)
 
 		if (!suppressInit && needsInit) data.init
 

codepulse/src/main/scala/com/secdec/codepulse/data/trace/slick/SlickMasterData.scala

@@ -0,0 +1,39 @@
+/*
+ * Code Pulse: A real-time code coverage testing tool. For more information
+ * see http://code-pulse.com
+ *
+ * Copyright (C) 2014 Applied Visions - http://securedecisions.avi.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.secdec.codepulse.data.trace.slick
+
+import scala.slick.driver.JdbcProfile
+import scala.slick.jdbc.JdbcBackend.Database
+
+/** Main access to centralized trace metadata, using Slick, with `db` and `driver`.
+  *
+  * @author robertf
+  */
+private[slick] class SlickMasterData(val db: Database, val driver: JdbcProfile) {
+	private val metadataDao = new TraceMetadataDao(driver)
+	private val metadataAccess = new SlickTraceMetadataMaster(metadataDao, db)
+
+	def metadataMaster: SlickTraceMetadataMaster = metadataAccess
+
+	/** Initialize a blank DB for use. */
+	private[slick] def init() = db withTransaction { implicit transaction =>
+		metadataDao.create
+	}
+}

codepulse/src/main/scala/com/secdec/codepulse/data/trace/slick/SlickTraceData.scala

@@ -31,10 +31,7 @@ import com.secdec.codepulse.data.trace._
   *
   * @author robertf
   */
-private[slick] class SlickTraceData(val db: Database, val driver: JdbcProfile, encounterBufferSize: Int, encounterFlushInterval: FiniteDuration, actorSystem: ActorSystem) extends TraceData {
-	private val traceMetadataDao = new TraceMetadataDao(driver)
-	private val metadataAccess = new SlickTraceMetadataAccess(traceMetadataDao, db) with DefaultTraceMetadataUpdates with TraceMetadata
-
+private[slick] class SlickTraceData(val db: Database, val driver: JdbcProfile, metadataAccess: SlickTraceMetadataAccess with TraceMetadata, encounterBufferSize: Int, encounterFlushInterval: FiniteDuration, actorSystem: ActorSystem) extends TraceData {
 	private val treeNodeDataDao = new TreeNodeDataDao(driver)
 	private val treeNodeDataAccess = new SlickTreeNodeDataAccess(treeNodeDataDao, db)
 
@@ -51,7 +48,6 @@ private[slick] class SlickTraceData(val db: Database, val driver: JdbcProfile, e
 
 	/** Initialize a blank DB for use. */
 	private[slick] def init() = db withTransaction { implicit transaction =>
-		traceMetadataDao.create
 		treeNodeDataDao.create
 		recordingMetadataDao.create
 		encountersDao.create
@@ -66,6 +62,8 @@ private[slick] class SlickTraceData(val db: Database, val driver: JdbcProfile, e
 	}
 
 	def delete() {
+		metadataAccess.delete
+
 		close
 
 		db withTransaction { implicit session =>

codepulse/src/main/scala/com/secdec/codepulse/data/trace/slick/SlickTraceMetadataAccess.scala

@@ -20,24 +20,38 @@
 package com.secdec.codepulse.data.trace.slick
 
 import scala.slick.jdbc.JdbcBackend.{ Database, Session }
-import com.secdec.codepulse.data.trace.TraceMetadataAccess
+import com.secdec.codepulse.data.trace.{ DefaultTraceMetadataUpdates, TraceMetadataAccess, TraceMetadata }
 import net.liftweb.util.Helpers.AsLong
 import net.liftweb.util.Helpers.AsBoolean
 
+/** Master controller for centralized Slick-backed metadata access.
+  *
+  * @author robertf
+  */
+private[slick] class SlickTraceMetadataMaster(dao: TraceMetadataDao, db: Database) {
+	def get(traceId: Int) = new SlickTraceMetadataAccess(traceId, dao, db) with DefaultTraceMetadataUpdates with TraceMetadata
+}
+
 /** Slick-backed TraceMetadataAccess implementation.
   *
   * @author robertf
   */
-private[slick] class SlickTraceMetadataAccess(dao: TraceMetadataDao, db: Database) extends TraceMetadataAccess {
+private[slick] class SlickTraceMetadataAccess(traceId: Int, dao: TraceMetadataDao, db: Database) extends TraceMetadataAccess {
 	private val cache = collection.mutable.Map.empty[String, Option[String]]
 
-	private def get(key: String)(implicit session: Session) = cache.getOrElseUpdate(key, dao get key)
+	private def get(key: String)(implicit session: Session) = cache.getOrElseUpdate(key, dao.get(traceId, key))
 	private def set(key: String, value: String)(implicit session: Session): Unit = set(key, Some(value))
 	private def set(key: String, value: Option[String])(implicit session: Session) {
-		dao.set(key, value)
+		dao.set(traceId, key, value)
 		cache += key -> value
 	}
 
+	def delete() {
+		db withSession { implicit session =>
+			dao delete traceId
+		}
+	}
+
 	def name = db withSession { implicit session =>
 		get("name")
 	} getOrElse "Untitled"

codepulse/src/main/scala/com/secdec/codepulse/data/trace/slick/TraceMetadataDao.scala

@@ -29,34 +29,42 @@ import com.secdec.codepulse.data.trace._
 private[slick] class TraceMetadataDao(val driver: JdbcProfile) {
 	import driver.simple._
 
-	class TraceMetadata(tag: Tag) extends Table[(String, String)](tag, "trace_metadata") {
-		def key = column[String]("key", O.PrimaryKey, O.NotNull)
+	class TraceMetadata(tag: Tag) extends Table[(Int, String, String)](tag, "trace_metadata") {
+		def traceId = column[Int]("trace_id", O.NotNull)
+		def key = column[String]("key", O.NotNull)
 		def value = column[String]("value", O.NotNull)
-		def * = key -> value
+		def pk = primaryKey("pk_trace_metadata", (traceId, key))
+		def * = (traceId, key, value)
 	}
 	val traceMetadata = TableQuery[TraceMetadata]
 
 	def create(implicit session: Session) = traceMetadata.ddl.create
 
-	def getMap()(implicit session: Session): Map[String, String] = traceMetadata.list.toMap
+	def getMap()(implicit session: Session): Map[Int, Map[String, String]] = traceMetadata.list.groupBy(_._1) mapValues {
+		entries => (entries map { case (_, key, value) => key -> value }).toMap
+	}
 
-	def get(key: String)(implicit session: Session): Option[String] =
-		(for (r <- traceMetadata if r.key === key) yield r.value).firstOption
+	def get(traceId: Int, key: String)(implicit session: Session): Option[String] =
+		(for (r <- traceMetadata if r.traceId === traceId && r.key === key) yield r.value).firstOption
 
-	def set(key: String, value: String)(implicit session: Session) {
-		set(key, Some(value))
+	def set(traceId: Int, key: String, value: String)(implicit session: Session) {
+		set(traceId, key, Some(value))
 	}
 
-	def set(key: String, value: Option[String])(implicit session: Session) {
+	def set(traceId: Int, key: String, value: Option[String])(implicit session: Session) {
 		value match {
 			case Some(value) =>
-				get(key) match {
-					case Some(_) => (for (r <- traceMetadata if r.key === key) yield r.value).update(value)
-					case None => traceMetadata += key -> value
+				get(traceId, key) match {
+					case Some(_) => (for (r <- traceMetadata if r.traceId === traceId && r.key === key) yield r.value).update(value)
+					case None => traceMetadata += (traceId, key, value)
 				}
 
 			case None =>
-				(for (r <- traceMetadata if r.key === key) yield r).delete
+				(for (r <- traceMetadata if r.traceId === traceId && r.key === key) yield r).delete
 		}
 	}
+
+	def delete(traceId: Int)(implicit session: Session) {
+		(for (r <- traceMetadata if r.traceId === traceId) yield r).delete
+	}
 }