Graph cache

build.sbt

@@ -517,6 +517,7 @@ lazy val webUtil = project
       Deps.kantanRegex.generic.value ::
       Deps.fontawesome.value ::
       ScalablyTyped.C.chart_dot_js ::
+      ScalablyTyped.B.`base64-js` ::
       Nil
   )
 
@@ -569,6 +570,7 @@ lazy val webApp = project
       Deps.npm.wdtEmojiBundle ::
       Deps.npm.tribute ::
       Deps.npm.chartJs ::
+      Deps.npm.base64Js ::
       Deps.npm.hopscotch ::
       Deps.npm.canvasImageUploader ::
       Deps.npm.exifJS ::

project/Deps.scala

@@ -162,6 +162,7 @@ object Deps {
     val wdtEmojiBundle = "wdt-emoji-bundle" -> "git://github.com/fdietze/wdt-emoji-bundle.git#fcf05d9"
     val tribute = "tributejs" -> "3.7.1"
     val chartJs = "chart.js" -> "2.8.0"
+    val base64Js = "base64-js" -> "1.3.1"
     val hopscotch = "hopscotch" -> "0.3.1"
     val canvasImageUploader = "canvas-image-uploader" -> "git+https://git@github.com/selbekk/CanvasImageUploader.git#6e5a71b5c2c01b00e76c86d65f2959d3fa9f3125" // fork without jquery and black canvas fix
     val exifJS = "exif-js" -> "git+https://git@github.com/fdietze/exif-js.git#da3116c" // fork merges PR which avoids runtime error

project/plugins.sbt

@@ -6,7 +6,7 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.19")
 
 // scalablytyped
 resolvers += Resolver.bintrayRepo("oyvindberg", "ScalablyTyped")
-addSbtPlugin("org.scalablytyped" % "sbt-scalablytyped" % "201911170530")
+addSbtPlugin("org.scalablytyped" % "sbt-scalablytyped" % "201911271221")
 
 // workflow
 addSbtPlugin("io.spray" % "sbt-revolver" % "0.9.1")

webApp/src/main/scala/wust/webApp/ClientStorage.scala

@@ -1,11 +1,14 @@
 package wust.webApp
 
+import scala.scalajs.js.typedarray._
 import wust.webApp.jsdom.Base64Codec
 import wust.api.serialize.Boopickle._
 import boopickle.Default._
 import wust.facades.jsSha256.Sha256
 import wust.util.Memo
 import wust.ids._
+import wust.graph.Graph
+import wust.graph.Page
 import collection.mutable
 import cats.effect.SyncIO
 import io.circe._
@@ -19,27 +22,40 @@ import wust.api.Authentication
 import wust.api.serialize.Circe._
 import wust.graph.GraphChanges
 import wust.webUtil.outwatchHelpers._
+import wust.webUtil.BinaryConvertors._
+import wust.util.time.time
 
-import scala.util.{Failure, Success, Try}
+import scala.util.{ Failure, Success, Try }
 import wust.facades.segment.Segment
+import scala.scalajs.js.typedarray.Uint8Array
+import java.nio.ByteBuffer
 
 class ClientStorage(implicit owner: Ctx.Owner) {
-  import org.scalajs.dom.ext.{LocalStorage => internal}
+  import org.scalajs.dom.ext.{ LocalStorage => internal }
 
   object keys {
     val auth = "wust.auth"
     val sidebarOpen = "wust.sidebar.open"
     val sidebarWithProjects = "wust.sidebar.projects"
     val taglistOpen = "wust.taglist.open"
     val filterlistOpen = "wust.filterlist.open"
-    def pendingChanges(userId:UserId) = s"wust.pendingchanges.${userId.toUuid.toString}"
+    def pendingChanges(userId: UserId) = s"wust.pendingchanges.${userId.toUuid.toString}"
     val backendTimeDelta = "wust.backendtimedelta"
+    def pageCache(page: Page) = new {
+      val baseKey = page.parentId match {
+        case Some(parentId) => s"wust.pageCache.${parentId.toUuid.toString}"
+        case None           => s"wust.pageCache.empty"
+      }
+      val graph = s"${baseKey}.graph"
+      val time = s"${baseKey}.time"
+      val keys = List(graph, time)
+    }
   }
 
   private def toJson[T: Encoder](value: T): String = value.asJson.noSpaces
   private def fromJson[T: Decoder](value: String): Option[T] = decode[T](value).right.toOption
 
-  val canAccessLs: Boolean = {
+  val canAccessStorage: Boolean = {
     Try {
       val woostStr = "wust-localstorage-write-test"
       internal.update(woostStr, woostStr)
@@ -54,8 +70,30 @@ class ClientStorage(implicit owner: Ctx.Owner) {
     }
   }
 
+  def getGraph(page: Page): Option[Graph] = {
+    val storageKey = keys.pageCache(page)
+    val encodedOpt = internal(storageKey.graph)
+    encodedOpt flatMap { encoded =>
+      decode[Graph](encoded) match {
+        case Left(_) =>
+          // cannot decode cache -> prune this entry
+          storageKey.keys.foreach(internal.remove)
+          None
+        case Right(decoded) => Some(decoded)
+      }
+    }
+  }
+
+  def updateGraph(page: Page, graph: Graph): Unit = {
+    time(s"writing to cache: $page") {
+      val storageKey = keys.pageCache(page)
+      internal.update(storageKey.graph, toJson(graph))
+      internal.update(storageKey.time, EpochMilli.now.toString)
+    }
+  }
+
   val auth: Var[Option[Authentication]] = {
-    if(canAccessLs) {
+    if (canAccessStorage) {
       LocalStorage
         .handlerWithoutEvents[SyncIO](keys.auth)
         .unsafeRunSync()
@@ -64,54 +102,65 @@ class ClientStorage(implicit owner: Ctx.Owner) {
     } else Var(None)
   }
 
-  private def graphChangeToHash(serialized:String):String = Sha256.sha224(serialized)
-  private def encodeBoopickleBase64(change: GraphChanges):String = Base64Codec.encode(Pickle.intoBytes(change))
-  private def decodeBoopickleBase64(encoded: String):Option[GraphChanges] = Try(Unpickle[GraphChanges].fromBytes(Base64Codec.decode(encoded))).toOption
 
-  def addPendingGraphChange(userId: UserId, change:GraphChanges) = {
-    val changeSerialized = encodeBoopickleBase64(change)
+  def compress(data: Uint8Array):Uint8Array = ???
+  private def encodeBoopickleBase64[T: Pickler](data: T): String = {
+    val serialized:ByteBuffer = Pickle.intoBytes(data)
+    val compressed = compress(serialized.toUint8Array)
+    Base64Codec.encode(compressed)
+  }
+  private def decodeBoopickleBase64[T: Pickler](encoded: String): Option[T] = Try(Unpickle[T].fromBytes(Base64Codec.decode(encoded))).toOption
+
+
+
+  private def graphChangeToHash(serialized: String): String = Sha256.sha224(serialized)
+  private def encodeGraphChangesBoopickleBase64(change: GraphChanges): String = Base64Codec.encode(Pickle.intoBytes(change).toUint8Array)
+  private def decodeGraphChangesBoopickleBase64(encoded: String): Option[GraphChanges] = Try(Unpickle[GraphChanges].fromBytes(Base64Codec.decode(encoded))).toOption
+
+  def addPendingGraphChange(userId: UserId, change: GraphChanges) = {
+    val changeSerialized = encodeGraphChangesBoopickleBase64(change)
     val key = graphChangeToHash(changeSerialized)
     pendingChanges(userId).update(_.updated(key, changeSerialized))
   }
 
-  def deletePendingGraphChanges(userId: UserId, changes:GraphChanges) = {
-    val changeSerialized = encodeBoopickleBase64(changes)
+  def deletePendingGraphChanges(userId: UserId, changes: GraphChanges) = {
+    val changeSerialized = encodeGraphChangesBoopickleBase64(changes)
     val key = graphChangeToHash(changeSerialized)
     pendingChanges(userId).update(_ - key)
   }
 
-  def getDecodablePendingGraphChanges(userId: UserId):List[GraphChanges] = {
+  def getDecodablePendingGraphChanges(userId: UserId): List[GraphChanges] = {
     val all = pendingChanges(userId).now
     val invalidKeysBuilder = mutable.ListBuffer.empty[String]
     val validDecodedBuilder = mutable.ListBuffer.empty[GraphChanges]
-    all.foreach { case(key, encoded) =>
-      decodeBoopickleBase64(encoded) match {
-        case None => 
-          invalidKeysBuilder += key
-          val errorId = NodeId.fresh().toUuid.toString
-          Segment.trackError("Failed to decode pending GraphChange", s"${errorId}")
-          Client.api.log(s"Failed to decode pending GraphChange: errorId=$errorId, change=$encoded")
-        case Some(valid) => 
-          validDecodedBuilder += valid
-      }
+    all.foreach {
+      case (key, encoded) =>
+        decodeGraphChangesBoopickleBase64(encoded) match {
+          case None =>
+            invalidKeysBuilder += key
+            val errorId = NodeId.fresh().toUuid.toString
+            Segment.trackError("Failed to decode pending GraphChange", s"${errorId}")
+            Client.api.log(s"Failed to decode pending GraphChange: errorId=$errorId, change=$encoded")
+          case Some(valid) =>
+            validDecodedBuilder += valid
+        }
     }
 
     val validDecoded = validDecodedBuilder.result()
 
-    if(all.size != validDecoded.size) {
+    if (all.size != validDecoded.size) {
       pendingChanges(userId).update(_ -- invalidKeysBuilder.result())
     }
 
     validDecoded
   }
 
-
   val pendingChanges = Memo.mutableHashMapMemo[UserId, Var[Map[String, String]]](pendingChangesByUser)
 
   //TODO: howto handle with events from other tabs?
   private def pendingChangesByUser(userId: UserId): Var[Map[String, String]] = {
     val storageKey = keys.pendingChanges(userId)
-    if(canAccessLs) {
+    if (canAccessStorage) {
       LocalStorage
         .handlerWithoutEvents[SyncIO](storageKey)
         .unsafeRunSync()
@@ -121,7 +170,7 @@ class ClientStorage(implicit owner: Ctx.Owner) {
   }
 
   val sidebarOpen: Var[Option[Boolean]] = {
-    if(canAccessLs) {
+    if (canAccessStorage) {
       LocalStorage
         .handlerWithoutEvents[SyncIO](keys.sidebarOpen)
         .unsafeRunSync()
@@ -131,7 +180,7 @@ class ClientStorage(implicit owner: Ctx.Owner) {
   }
 
   val sidebarWithProjects: Var[Option[Boolean]] = {
-    if(canAccessLs) {
+    if (canAccessStorage) {
       LocalStorage
         .handlerWithoutEvents[SyncIO](keys.sidebarWithProjects)
         .unsafeRunSync()
@@ -141,7 +190,7 @@ class ClientStorage(implicit owner: Ctx.Owner) {
   }
 
   val taglistOpen: Var[Option[Boolean]] = {
-    if(canAccessLs) {
+    if (canAccessStorage) {
       LocalStorage
         .handlerWithoutEvents[SyncIO](keys.taglistOpen)
         .unsafeRunSync()
@@ -151,7 +200,7 @@ class ClientStorage(implicit owner: Ctx.Owner) {
   }
 
   val filterlistOpen: Var[Option[Boolean]] = {
-    if(canAccessLs) {
+    if (canAccessStorage) {
       LocalStorage
         .handlerWithoutEvents[SyncIO](keys.filterlistOpen)
         .unsafeRunSync()
@@ -161,7 +210,7 @@ class ClientStorage(implicit owner: Ctx.Owner) {
   }
 
   val backendTimeDelta: Var[DurationMilli] = {
-    if(canAccessLs) {
+    if (canAccessStorage) {
       LocalStorage
         .handlerWithoutEvents[SyncIO](keys.backendTimeDelta)
         .unsafeRunSync()

webApp/src/main/scala/wust/webApp/jsdom/Base64Codec.scala

@@ -5,20 +5,15 @@ import java.nio.ByteBuffer
 import org.scalajs.dom.window.{atob, btoa}
 
 import scala.scalajs.js
+import scala.scalajs.js.typedarray.Uint8Array
+import typings.base64DashJs._
+import scala.scalajs.js.typedarray
+import scala.scalajs.js.typedarray.byteArray2Int8Array
 
 object Base64Codec {
   import js.Dynamic.{global => g}
 
-  def encode(buffer: ByteBuffer): String = {
-    val n = buffer.limit()
-    val s = new StringBuilder(n)
-    for (_ <- 0 until n) {
-      val c = buffer.get
-      s ++= g.String.fromCharCode(c & 0xFF).asInstanceOf[String]
-    }
-
-    btoa(s.result)
-  }
+  def encode(bytes: Uint8Array):String = base64DashJsMod.fromByteArray(bytes.asInstanceOf[typings.std.Uint8Array])
 
   def decode(data: String): ByteBuffer = {
     // remove urlsafety first:

webApp/src/main/scala/wust/webApp/jsdom/Notifications.scala

@@ -4,6 +4,7 @@ import org.scalajs.dom.experimental
 import org.scalajs.dom.experimental.NotificationOptions
 import org.scalajs.dom.experimental.permissions._
 import org.scalajs.dom.experimental.push._
+import wust.webUtil.BinaryConvertors._
 import rx._
 import wust.api._
 import wust.webApp.Client
@@ -140,9 +141,9 @@ object Notifications {
                 val webpush = WebPushSubscription(
                   endpointUrl = sub.endpoint,
                   p256dh = Base64Codec.encode(
-                    TypedArrayBuffer.wrap(sub.getKey(PushEncryptionKeyName.p256dh))),
+                    TypedArrayBuffer.wrap(sub.getKey(PushEncryptionKeyName.p256dh).toUint8Array)),
                   auth = Base64Codec.encode(
-                    TypedArrayBuffer.wrap(sub.getKey(PushEncryptionKeyName.auth))))
+                    TypedArrayBuffer.wrap(sub.getKey(PushEncryptionKeyName.auth).toUint8Array)))
                 scribe.info(s"WebPush subscription: $webpush")
                 sendSubscription(webpush)
               case err =>

webApp/src/main/scala/wust/webApp/state/GlobalState.scala

@@ -1,6 +1,7 @@
 package wust.webApp.state
 
 // import acyclic.file
+import wust.webApp.ClientStorage
 import com.github.ghik.silencer.silent
 import org.scalajs.dom.experimental.permissions.PermissionState
 import org.scalajs.dom.window
@@ -19,6 +20,7 @@ import wust.webApp.views._
 import wust.webApp.{ Client, WoostConfig }
 import wust.webUtil.outwatchHelpers._
 import wust.webUtil.{ BrowserDetect, ModalConfig, Ownable }
+import wust.webUtil.Elements.defer
 import wust.facades.segment.Segment
 import outwatch.dom._
 import outwatch.dom.dsl._
@@ -178,16 +180,23 @@ object GlobalState {
   val pageExistsInGraph: Rx[Boolean] = Rx{ page().parentId.exists(rawGraph().contains) }
   val showPageNotFound = Rx { !isLoading() && !pageExistsInGraph() && viewIsContent() }
 
-  def focus(nodeId: NodeId, needsGet: Boolean = true) = {
-    val alreadyLoaded = (
+  def focus(nodeId: NodeId, view: Option[View] = GlobalState.urlConfig.now.view, needsGet: Boolean = true): Unit = focusPage(Page(nodeId), needsGet = needsGet)
+  def focusPage(page: Page, view: Option[View] = GlobalState.urlConfig.now.view, needsGet: Boolean = true): Unit = {
+    @inline def nextPage = page
+    val oldPage = GlobalState.page.now
+    val oldGraph = graph.now
+    val oldRawGraph = rawGraph.now
+    def alreadyLoaded = (
       for {
-        pageId <- page.now.parentId
-        pageIdx <- graph.now.idToIdx(pageId)
-        nodeIdx <- graph.now.idToIdx(nodeId)
-      } yield dfs.exists(_(pageIdx), dfs.withStart, graph.now.childrenIdx, isFound = { _ == nodeIdx })
+        oldPageId <- oldPage.parentId
+        oldPageIdx <- oldGraph.idToIdx(oldPageId)
+        nextPageId <- nextPage.parentId
+        nextPageIdx <- oldGraph.idToIdx(nextPageId)
+      } yield dfs.exists(_(oldPageIdx), dfs.withStart, oldGraph.childrenIdx, isFound = { _ == nextPageIdx })
     ).getOrElse(false)
 
-    urlConfig.update(_.focus(Page(nodeId), needsGet = needsGet && !alreadyLoaded))
+    defer { Client.storage.updateGraph(oldPage, oldRawGraph) }
+    urlConfig.update(_.focus(nextPage, view, needsGet = needsGet && !alreadyLoaded))
   }
 
   def focusSubPage(nodeIdOpt: Option[NodeId]) = {

webApp/src/main/scala/wust/webApp/state/UrlConfig.scala

@@ -42,7 +42,7 @@ final case class UrlConfig(
   @inline def focus(page: Page, view: View): UrlConfig = focus(page, Some(view))
   @inline def focus(view: Option[View]): UrlConfig = copy(view = view, redirectTo = None, focusId = None)
   @inline def focus(page: Page, view: View, needsGet: Boolean): UrlConfig = focus(page, Some(view), needsGet)
-  def focus(page: Page, view: Option[View] = None, needsGet: Boolean = true): UrlConfig = copy(pageChange = PageChange(page, needsGet = needsGet), view = view, redirectTo = None, focusId = None, subPage = Page.empty)
+  @inline def focus(page: Page, view: Option[View] = None, needsGet: Boolean = true): UrlConfig = copy(pageChange = PageChange(page, needsGet = needsGet), view = view, redirectTo = None, focusId = None, subPage = Page.empty)
 
 }
 

webApp/src/main/scala/wust/webApp/views/AuthControls.scala

@@ -90,7 +90,7 @@ object AuthControls {
       cls := s"tiny compact ui $buttonStyle button",
       onClick foreach {
         Client.auth.logout().foreach { _ =>
-          GlobalState.urlConfig.update(_.focus(Page.empty, View.Login))
+          GlobalState.focusPage(Page.empty, view = Some(View.Login))
         }
         FeatureState.use(Feature.ClickLogoutInAuthStatus)
         Segment.trackSignedOut()

webApp/src/main/scala/wust/webApp/views/Components.scala

@@ -152,7 +152,7 @@ object Components {
           } // Max 1 dm node with this name
           previousDmNode match {
             case Some(dmNode) if graph.can_access_node(user.id, dmNode.id) =>
-              GlobalState.urlConfig.update(_.focus(Page(dmNode.id), View.Conversation))
+              GlobalState.focus(dmNode.id, view = Some(View.Conversation))
             case _ => // create a new channel, add user as member
               val nodeId = NodeId.fresh
               val change:GraphChanges =
@@ -164,7 +164,7 @@ object Components {
                 ))
 
               GlobalState.submitChanges(change)
-              GlobalState.urlConfig.update(_.focus(Page(nodeId), View.Chat, needsGet = false))
+              GlobalState.focus(nodeId, Some(View.Chat), needsGet = false)
               ()
           }
           Segment.trackEvent("Direct Message")

webApp/src/main/scala/wust/webApp/views/CreateNewPrompt.scala

@@ -72,7 +72,7 @@ object CreateNewPrompt {
       if (addToChannels.now) {
         val channelChanges = GraphChanges.connect(Edge.Pinned)(newNode.id, GlobalState.user.now.id)
         GlobalState.submitChanges(changes merge channelChanges)
-        GlobalState.urlConfig.update(_.focus(Page(newNode.id), needsGet = false))
+        GlobalState.focus(newNode.id, needsGet = false)
       } else {
         GlobalState.submitChanges(changes).foreach { _ =>
           if(childNodes.now.nonEmpty)