diff --git a/build.sbt b/build.sbt index a9881dedc..d6fb62fa4 100644 --- a/build.sbt +++ b/build.sbt @@ -517,6 +517,7 @@ lazy val webUtil = project Deps.kantanRegex.generic.value :: Deps.fontawesome.value :: ScalablyTyped.C.chart_dot_js :: + ScalablyTyped.B.`base64-js` :: Nil ) @@ -569,6 +570,7 @@ lazy val webApp = project Deps.npm.wdtEmojiBundle :: Deps.npm.tribute :: Deps.npm.chartJs :: + Deps.npm.base64Js :: Deps.npm.hopscotch :: Deps.npm.canvasImageUploader :: Deps.npm.exifJS :: diff --git a/project/Deps.scala b/project/Deps.scala index 157e2d7bc..7cebeade6 100644 --- a/project/Deps.scala +++ b/project/Deps.scala @@ -162,6 +162,7 @@ object Deps { val wdtEmojiBundle = "wdt-emoji-bundle" -> "git://github.com/fdietze/wdt-emoji-bundle.git#fcf05d9" val tribute = "tributejs" -> "3.7.1" val chartJs = "chart.js" -> "2.8.0" + val base64Js = "base64-js" -> "1.3.1" val hopscotch = "hopscotch" -> "0.3.1" val canvasImageUploader = "canvas-image-uploader" -> "git+https://git@github.com/selbekk/CanvasImageUploader.git#6e5a71b5c2c01b00e76c86d65f2959d3fa9f3125" // fork without jquery and black canvas fix val exifJS = "exif-js" -> "git+https://git@github.com/fdietze/exif-js.git#da3116c" // fork merges PR which avoids runtime error diff --git a/project/plugins.sbt b/project/plugins.sbt index 8d40a781f..f317a040e 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -6,7 +6,7 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.19") // scalablytyped resolvers += Resolver.bintrayRepo("oyvindberg", "ScalablyTyped") -addSbtPlugin("org.scalablytyped" % "sbt-scalablytyped" % "201911170530") +addSbtPlugin("org.scalablytyped" % "sbt-scalablytyped" % "201911271221") // workflow addSbtPlugin("io.spray" % "sbt-revolver" % "0.9.1") diff --git a/webApp/src/main/scala/wust/webApp/ClientStorage.scala b/webApp/src/main/scala/wust/webApp/ClientStorage.scala index 7f43b1751..98aef5553 100644 --- a/webApp/src/main/scala/wust/webApp/ClientStorage.scala +++ b/webApp/src/main/scala/wust/webApp/ClientStorage.scala @@ -1,11 +1,14 @@ package wust.webApp +import scala.scalajs.js.typedarray._ import wust.webApp.jsdom.Base64Codec import wust.api.serialize.Boopickle._ import boopickle.Default._ import wust.facades.jsSha256.Sha256 import wust.util.Memo import wust.ids._ +import wust.graph.Graph +import wust.graph.Page import collection.mutable import cats.effect.SyncIO import io.circe._ @@ -19,12 +22,16 @@ import wust.api.Authentication import wust.api.serialize.Circe._ import wust.graph.GraphChanges import wust.webUtil.outwatchHelpers._ +import wust.webUtil.BinaryConvertors._ +import wust.util.time.time -import scala.util.{Failure, Success, Try} +import scala.util.{ Failure, Success, Try } import wust.facades.segment.Segment +import scala.scalajs.js.typedarray.Uint8Array +import java.nio.ByteBuffer class ClientStorage(implicit owner: Ctx.Owner) { - import org.scalajs.dom.ext.{LocalStorage => internal} + import org.scalajs.dom.ext.{ LocalStorage => internal } object keys { val auth = "wust.auth" @@ -32,14 +39,23 @@ class ClientStorage(implicit owner: Ctx.Owner) { val sidebarWithProjects = "wust.sidebar.projects" val taglistOpen = "wust.taglist.open" val filterlistOpen = "wust.filterlist.open" - def pendingChanges(userId:UserId) = s"wust.pendingchanges.${userId.toUuid.toString}" + def pendingChanges(userId: UserId) = s"wust.pendingchanges.${userId.toUuid.toString}" val backendTimeDelta = "wust.backendtimedelta" + def pageCache(page: Page) = new { + val baseKey = page.parentId match { + case Some(parentId) => s"wust.pageCache.${parentId.toUuid.toString}" + case None => s"wust.pageCache.empty" + } + val graph = s"${baseKey}.graph" + val time = s"${baseKey}.time" + val keys = List(graph, time) + } } private def toJson[T: Encoder](value: T): String = value.asJson.noSpaces private def fromJson[T: Decoder](value: String): Option[T] = decode[T](value).right.toOption - val canAccessLs: Boolean = { + val canAccessStorage: Boolean = { Try { val woostStr = "wust-localstorage-write-test" internal.update(woostStr, woostStr) @@ -54,8 +70,30 @@ class ClientStorage(implicit owner: Ctx.Owner) { } } + def getGraph(page: Page): Option[Graph] = { + val storageKey = keys.pageCache(page) + val encodedOpt = internal(storageKey.graph) + encodedOpt flatMap { encoded => + decode[Graph](encoded) match { + case Left(_) => + // cannot decode cache -> prune this entry + storageKey.keys.foreach(internal.remove) + None + case Right(decoded) => Some(decoded) + } + } + } + + def updateGraph(page: Page, graph: Graph): Unit = { + time(s"writing to cache: $page") { + val storageKey = keys.pageCache(page) + internal.update(storageKey.graph, toJson(graph)) + internal.update(storageKey.time, EpochMilli.now.toString) + } + } + val auth: Var[Option[Authentication]] = { - if(canAccessLs) { + if (canAccessStorage) { LocalStorage .handlerWithoutEvents[SyncIO](keys.auth) .unsafeRunSync() @@ -64,54 +102,65 @@ class ClientStorage(implicit owner: Ctx.Owner) { } else Var(None) } - private def graphChangeToHash(serialized:String):String = Sha256.sha224(serialized) - private def encodeBoopickleBase64(change: GraphChanges):String = Base64Codec.encode(Pickle.intoBytes(change)) - private def decodeBoopickleBase64(encoded: String):Option[GraphChanges] = Try(Unpickle[GraphChanges].fromBytes(Base64Codec.decode(encoded))).toOption - def addPendingGraphChange(userId: UserId, change:GraphChanges) = { - val changeSerialized = encodeBoopickleBase64(change) + def compress(data: Uint8Array):Uint8Array = ??? + private def encodeBoopickleBase64[T: Pickler](data: T): String = { + val serialized:ByteBuffer = Pickle.intoBytes(data) + val compressed = compress(serialized.toUint8Array) + Base64Codec.encode(compressed) + } + private def decodeBoopickleBase64[T: Pickler](encoded: String): Option[T] = Try(Unpickle[T].fromBytes(Base64Codec.decode(encoded))).toOption + + + + private def graphChangeToHash(serialized: String): String = Sha256.sha224(serialized) + private def encodeGraphChangesBoopickleBase64(change: GraphChanges): String = Base64Codec.encode(Pickle.intoBytes(change).toUint8Array) + private def decodeGraphChangesBoopickleBase64(encoded: String): Option[GraphChanges] = Try(Unpickle[GraphChanges].fromBytes(Base64Codec.decode(encoded))).toOption + + def addPendingGraphChange(userId: UserId, change: GraphChanges) = { + val changeSerialized = encodeGraphChangesBoopickleBase64(change) val key = graphChangeToHash(changeSerialized) pendingChanges(userId).update(_.updated(key, changeSerialized)) } - def deletePendingGraphChanges(userId: UserId, changes:GraphChanges) = { - val changeSerialized = encodeBoopickleBase64(changes) + def deletePendingGraphChanges(userId: UserId, changes: GraphChanges) = { + val changeSerialized = encodeGraphChangesBoopickleBase64(changes) val key = graphChangeToHash(changeSerialized) pendingChanges(userId).update(_ - key) } - def getDecodablePendingGraphChanges(userId: UserId):List[GraphChanges] = { + def getDecodablePendingGraphChanges(userId: UserId): List[GraphChanges] = { val all = pendingChanges(userId).now val invalidKeysBuilder = mutable.ListBuffer.empty[String] val validDecodedBuilder = mutable.ListBuffer.empty[GraphChanges] - all.foreach { case(key, encoded) => - decodeBoopickleBase64(encoded) match { - case None => - invalidKeysBuilder += key - val errorId = NodeId.fresh().toUuid.toString - Segment.trackError("Failed to decode pending GraphChange", s"${errorId}") - Client.api.log(s"Failed to decode pending GraphChange: errorId=$errorId, change=$encoded") - case Some(valid) => - validDecodedBuilder += valid - } + all.foreach { + case (key, encoded) => + decodeGraphChangesBoopickleBase64(encoded) match { + case None => + invalidKeysBuilder += key + val errorId = NodeId.fresh().toUuid.toString + Segment.trackError("Failed to decode pending GraphChange", s"${errorId}") + Client.api.log(s"Failed to decode pending GraphChange: errorId=$errorId, change=$encoded") + case Some(valid) => + validDecodedBuilder += valid + } } val validDecoded = validDecodedBuilder.result() - if(all.size != validDecoded.size) { + if (all.size != validDecoded.size) { pendingChanges(userId).update(_ -- invalidKeysBuilder.result()) } validDecoded } - val pendingChanges = Memo.mutableHashMapMemo[UserId, Var[Map[String, String]]](pendingChangesByUser) //TODO: howto handle with events from other tabs? private def pendingChangesByUser(userId: UserId): Var[Map[String, String]] = { val storageKey = keys.pendingChanges(userId) - if(canAccessLs) { + if (canAccessStorage) { LocalStorage .handlerWithoutEvents[SyncIO](storageKey) .unsafeRunSync() @@ -121,7 +170,7 @@ class ClientStorage(implicit owner: Ctx.Owner) { } val sidebarOpen: Var[Option[Boolean]] = { - if(canAccessLs) { + if (canAccessStorage) { LocalStorage .handlerWithoutEvents[SyncIO](keys.sidebarOpen) .unsafeRunSync() @@ -131,7 +180,7 @@ class ClientStorage(implicit owner: Ctx.Owner) { } val sidebarWithProjects: Var[Option[Boolean]] = { - if(canAccessLs) { + if (canAccessStorage) { LocalStorage .handlerWithoutEvents[SyncIO](keys.sidebarWithProjects) .unsafeRunSync() @@ -141,7 +190,7 @@ class ClientStorage(implicit owner: Ctx.Owner) { } val taglistOpen: Var[Option[Boolean]] = { - if(canAccessLs) { + if (canAccessStorage) { LocalStorage .handlerWithoutEvents[SyncIO](keys.taglistOpen) .unsafeRunSync() @@ -151,7 +200,7 @@ class ClientStorage(implicit owner: Ctx.Owner) { } val filterlistOpen: Var[Option[Boolean]] = { - if(canAccessLs) { + if (canAccessStorage) { LocalStorage .handlerWithoutEvents[SyncIO](keys.filterlistOpen) .unsafeRunSync() @@ -161,7 +210,7 @@ class ClientStorage(implicit owner: Ctx.Owner) { } val backendTimeDelta: Var[DurationMilli] = { - if(canAccessLs) { + if (canAccessStorage) { LocalStorage .handlerWithoutEvents[SyncIO](keys.backendTimeDelta) .unsafeRunSync() diff --git a/webApp/src/main/scala/wust/webApp/jsdom/Base64Codec.scala b/webApp/src/main/scala/wust/webApp/jsdom/Base64Codec.scala index f1c78405f..0371455b8 100644 --- a/webApp/src/main/scala/wust/webApp/jsdom/Base64Codec.scala +++ b/webApp/src/main/scala/wust/webApp/jsdom/Base64Codec.scala @@ -5,20 +5,15 @@ import java.nio.ByteBuffer import org.scalajs.dom.window.{atob, btoa} import scala.scalajs.js +import scala.scalajs.js.typedarray.Uint8Array +import typings.base64DashJs._ +import scala.scalajs.js.typedarray +import scala.scalajs.js.typedarray.byteArray2Int8Array object Base64Codec { import js.Dynamic.{global => g} - def encode(buffer: ByteBuffer): String = { - val n = buffer.limit() - val s = new StringBuilder(n) - for (_ <- 0 until n) { - val c = buffer.get - s ++= g.String.fromCharCode(c & 0xFF).asInstanceOf[String] - } - - btoa(s.result) - } + def encode(bytes: Uint8Array):String = base64DashJsMod.fromByteArray(bytes.asInstanceOf[typings.std.Uint8Array]) def decode(data: String): ByteBuffer = { // remove urlsafety first: diff --git a/webApp/src/main/scala/wust/webApp/jsdom/Notifications.scala b/webApp/src/main/scala/wust/webApp/jsdom/Notifications.scala index c5f5cbf5d..4c35e1bc2 100644 --- a/webApp/src/main/scala/wust/webApp/jsdom/Notifications.scala +++ b/webApp/src/main/scala/wust/webApp/jsdom/Notifications.scala @@ -4,6 +4,7 @@ import org.scalajs.dom.experimental import org.scalajs.dom.experimental.NotificationOptions import org.scalajs.dom.experimental.permissions._ import org.scalajs.dom.experimental.push._ +import wust.webUtil.BinaryConvertors._ import rx._ import wust.api._ import wust.webApp.Client @@ -140,9 +141,9 @@ object Notifications { val webpush = WebPushSubscription( endpointUrl = sub.endpoint, p256dh = Base64Codec.encode( - TypedArrayBuffer.wrap(sub.getKey(PushEncryptionKeyName.p256dh))), + TypedArrayBuffer.wrap(sub.getKey(PushEncryptionKeyName.p256dh).toUint8Array)), auth = Base64Codec.encode( - TypedArrayBuffer.wrap(sub.getKey(PushEncryptionKeyName.auth)))) + TypedArrayBuffer.wrap(sub.getKey(PushEncryptionKeyName.auth).toUint8Array))) scribe.info(s"WebPush subscription: $webpush") sendSubscription(webpush) case err => diff --git a/webApp/src/main/scala/wust/webApp/state/GlobalState.scala b/webApp/src/main/scala/wust/webApp/state/GlobalState.scala index 122e94c20..3b5fbf3e3 100644 --- a/webApp/src/main/scala/wust/webApp/state/GlobalState.scala +++ b/webApp/src/main/scala/wust/webApp/state/GlobalState.scala @@ -1,6 +1,7 @@ package wust.webApp.state // import acyclic.file +import wust.webApp.ClientStorage import com.github.ghik.silencer.silent import org.scalajs.dom.experimental.permissions.PermissionState import org.scalajs.dom.window @@ -19,6 +20,7 @@ import wust.webApp.views._ import wust.webApp.{ Client, WoostConfig } import wust.webUtil.outwatchHelpers._ import wust.webUtil.{ BrowserDetect, ModalConfig, Ownable } +import wust.webUtil.Elements.defer import wust.facades.segment.Segment import outwatch.dom._ import outwatch.dom.dsl._ @@ -178,16 +180,23 @@ object GlobalState { val pageExistsInGraph: Rx[Boolean] = Rx{ page().parentId.exists(rawGraph().contains) } val showPageNotFound = Rx { !isLoading() && !pageExistsInGraph() && viewIsContent() } - def focus(nodeId: NodeId, needsGet: Boolean = true) = { - val alreadyLoaded = ( + def focus(nodeId: NodeId, view: Option[View] = GlobalState.urlConfig.now.view, needsGet: Boolean = true): Unit = focusPage(Page(nodeId), needsGet = needsGet) + def focusPage(page: Page, view: Option[View] = GlobalState.urlConfig.now.view, needsGet: Boolean = true): Unit = { + @inline def nextPage = page + val oldPage = GlobalState.page.now + val oldGraph = graph.now + val oldRawGraph = rawGraph.now + def alreadyLoaded = ( for { - pageId <- page.now.parentId - pageIdx <- graph.now.idToIdx(pageId) - nodeIdx <- graph.now.idToIdx(nodeId) - } yield dfs.exists(_(pageIdx), dfs.withStart, graph.now.childrenIdx, isFound = { _ == nodeIdx }) + oldPageId <- oldPage.parentId + oldPageIdx <- oldGraph.idToIdx(oldPageId) + nextPageId <- nextPage.parentId + nextPageIdx <- oldGraph.idToIdx(nextPageId) + } yield dfs.exists(_(oldPageIdx), dfs.withStart, oldGraph.childrenIdx, isFound = { _ == nextPageIdx }) ).getOrElse(false) - urlConfig.update(_.focus(Page(nodeId), needsGet = needsGet && !alreadyLoaded)) + defer { Client.storage.updateGraph(oldPage, oldRawGraph) } + urlConfig.update(_.focus(nextPage, view, needsGet = needsGet && !alreadyLoaded)) } def focusSubPage(nodeIdOpt: Option[NodeId]) = { diff --git a/webApp/src/main/scala/wust/webApp/state/UrlConfig.scala b/webApp/src/main/scala/wust/webApp/state/UrlConfig.scala index fa491a27b..e60f1a353 100644 --- a/webApp/src/main/scala/wust/webApp/state/UrlConfig.scala +++ b/webApp/src/main/scala/wust/webApp/state/UrlConfig.scala @@ -42,7 +42,7 @@ final case class UrlConfig( @inline def focus(page: Page, view: View): UrlConfig = focus(page, Some(view)) @inline def focus(view: Option[View]): UrlConfig = copy(view = view, redirectTo = None, focusId = None) @inline def focus(page: Page, view: View, needsGet: Boolean): UrlConfig = focus(page, Some(view), needsGet) - def focus(page: Page, view: Option[View] = None, needsGet: Boolean = true): UrlConfig = copy(pageChange = PageChange(page, needsGet = needsGet), view = view, redirectTo = None, focusId = None, subPage = Page.empty) + @inline def focus(page: Page, view: Option[View] = None, needsGet: Boolean = true): UrlConfig = copy(pageChange = PageChange(page, needsGet = needsGet), view = view, redirectTo = None, focusId = None, subPage = Page.empty) } diff --git a/webApp/src/main/scala/wust/webApp/views/AuthControls.scala b/webApp/src/main/scala/wust/webApp/views/AuthControls.scala index ee0b11570..7daef7188 100644 --- a/webApp/src/main/scala/wust/webApp/views/AuthControls.scala +++ b/webApp/src/main/scala/wust/webApp/views/AuthControls.scala @@ -90,7 +90,7 @@ object AuthControls { cls := s"tiny compact ui $buttonStyle button", onClick foreach { Client.auth.logout().foreach { _ => - GlobalState.urlConfig.update(_.focus(Page.empty, View.Login)) + GlobalState.focusPage(Page.empty, view = Some(View.Login)) } FeatureState.use(Feature.ClickLogoutInAuthStatus) Segment.trackSignedOut() diff --git a/webApp/src/main/scala/wust/webApp/views/Components.scala b/webApp/src/main/scala/wust/webApp/views/Components.scala index c402b905d..29d65b7f9 100644 --- a/webApp/src/main/scala/wust/webApp/views/Components.scala +++ b/webApp/src/main/scala/wust/webApp/views/Components.scala @@ -152,7 +152,7 @@ object Components { } // Max 1 dm node with this name previousDmNode match { case Some(dmNode) if graph.can_access_node(user.id, dmNode.id) => - GlobalState.urlConfig.update(_.focus(Page(dmNode.id), View.Conversation)) + GlobalState.focus(dmNode.id, view = Some(View.Conversation)) case _ => // create a new channel, add user as member val nodeId = NodeId.fresh val change:GraphChanges = @@ -164,7 +164,7 @@ object Components { )) GlobalState.submitChanges(change) - GlobalState.urlConfig.update(_.focus(Page(nodeId), View.Chat, needsGet = false)) + GlobalState.focus(nodeId, Some(View.Chat), needsGet = false) () } Segment.trackEvent("Direct Message") diff --git a/webApp/src/main/scala/wust/webApp/views/CreateNewPrompt.scala b/webApp/src/main/scala/wust/webApp/views/CreateNewPrompt.scala index eaac68a0e..66c76350c 100644 --- a/webApp/src/main/scala/wust/webApp/views/CreateNewPrompt.scala +++ b/webApp/src/main/scala/wust/webApp/views/CreateNewPrompt.scala @@ -72,7 +72,7 @@ object CreateNewPrompt { if (addToChannels.now) { val channelChanges = GraphChanges.connect(Edge.Pinned)(newNode.id, GlobalState.user.now.id) GlobalState.submitChanges(changes merge channelChanges) - GlobalState.urlConfig.update(_.focus(Page(newNode.id), needsGet = false)) + GlobalState.focus(newNode.id, needsGet = false) } else { GlobalState.submitChanges(changes).foreach { _ => if(childNodes.now.nonEmpty) diff --git a/webApp/src/main/scala/wust/webApp/views/DoodleView.scala b/webApp/src/main/scala/wust/webApp/views/DoodleView.scala index 52dd476a8..c7799aee9 100644 --- a/webApp/src/main/scala/wust/webApp/views/DoodleView.scala +++ b/webApp/src/main/scala/wust/webApp/views/DoodleView.scala @@ -270,7 +270,7 @@ object DoodleView extends AppDefinition { )).foreach { val nodeId = NodeId.fresh GlobalState.submitChanges(createNode(nodeId)) - GlobalState.urlConfig.update(_.focus(Page(nodeId), needsGet = false)) + GlobalState.focus(nodeId, needsGet = false) () } diff --git a/webApp/src/main/scala/wust/webApp/views/MembersModal.scala b/webApp/src/main/scala/wust/webApp/views/MembersModal.scala index 7f254f6d2..988b00e78 100644 --- a/webApp/src/main/scala/wust/webApp/views/MembersModal.scala +++ b/webApp/src/main/scala/wust/webApp/views/MembersModal.scala @@ -121,7 +121,7 @@ object MembersModal { if (membership.userId == GlobalState.user.now.id) { needAction() = Some(NeedAction( { () => - GlobalState.urlConfig.update(_.focus(Page.empty)) + GlobalState.focusPage(Page.empty) GlobalState.uiModalClose.onNext(()) action() }, diff --git a/webApp/src/main/scala/wust/webApp/views/NewProjectPrompt.scala b/webApp/src/main/scala/wust/webApp/views/NewProjectPrompt.scala index 8ffbc0305..187d67264 100644 --- a/webApp/src/main/scala/wust/webApp/views/NewProjectPrompt.scala +++ b/webApp/src/main/scala/wust/webApp/views/NewProjectPrompt.scala @@ -52,7 +52,7 @@ object NewProjectPrompt { val views = if (selectedViews.now.isEmpty) None else Some(selectedViews.now.toList) GlobalState.submitChanges(GraphChanges.newProject(nodeId, GlobalState.user.now.id, newName, views) merge sub.changes(nodeId) merge extraChanges(nodeId)) - if (focusNewProject) GlobalState.urlConfig.update(_.focus(Page(nodeId), needsGet = false)) + if (focusNewProject) GlobalState.focus(nodeId, needsGet = false) FeatureState.use(Feature.CreateProject) selectedViews.now.foreach (ViewModificationMenu.trackAddViewFeature) diff --git a/webUtil/src/main/scala/wust/webUtil/BinaryConvertors.scala b/webUtil/src/main/scala/wust/webUtil/BinaryConvertors.scala new file mode 100644 index 000000000..bb32182d5 --- /dev/null +++ b/webUtil/src/main/scala/wust/webUtil/BinaryConvertors.scala @@ -0,0 +1,24 @@ +package wust.webUtil + +import java.nio.ByteBuffer + +import org.scalajs.dom.window.{atob, btoa} + +import scala.scalajs.js +import scala.scalajs.js.typedarray.Uint8Array +import typings.base64DashJs._ +import scala.scalajs.js.typedarray +import scala.scalajs.js.typedarray.byteArray2Int8Array + +object BinaryConvertors { + implicit class RichTypedArrayArrayBuffer(val bytes: typedarray.ArrayBuffer) extends AnyVal { + // TODO: TypedArrayBuffer.wrap() ? + @inline def toUint8Array = new Uint8Array(bytes) + } + implicit class RichArrayOfByte(val bytes: Array[Byte]) extends AnyVal { + @inline def toUint8Array = new Uint8Array(byteArray2Int8Array(bytes)) + } + implicit class RichJavaNioByteBuffer(val bytes: java.nio.ByteBuffer) extends AnyVal { + @inline def toUint8Array = bytes.array.toUint8Array + } +}