Refactor and fix eval channels (#2302)

* wip

* pull out eval into the interpreter

* fix web eval

* fix DioxusChannel name

* properly drop dioxus channel

* use typescript dioxus chanel in desktop

* add more comments to native eval

* add desktop headless eval tests

* expand web playwright eval tests

* fix web headless tests

* fix default hasher path

* run eval tests on windows

* restore desktop query drop code

* remove data from drop desktop query message

* catch syntax errors in desktop eval

* catch js runtime errors in desktop

* fix typo interprerter -> interpreter

---------

Co-authored-by: Jonathan Kelley <jkelleyrtp@gmail.com>

packages/interpreter/Cargo.toml

@@ -12,6 +12,7 @@ keywords = ["dom", "ui", "gui", "react", "wasm"]
 
 [dependencies]
 wasm-bindgen = { workspace = true, optional = true }
+wasm-bindgen-futures = { workspace = true, optional = true }
 js-sys = { version = "0.3.56", optional = true }
 web-sys = { version = "0.3.56", optional = true, features = [
     "Element",
@@ -34,9 +35,11 @@ sledgehammer = ["sledgehammer_bindgen", "sledgehammer_utils"]
 webonly = [
     "sledgehammer",
     "wasm-bindgen",
+    "wasm-bindgen-futures",
     "js-sys",
     "web-sys",
     "sledgehammer_bindgen/web",
 ]
 binary-protocol = ["sledgehammer", "dioxus-core", "dioxus-html"]
 minimal_bindings = []
+eval = []

packages/interpreter/build.rs

@@ -1,4 +1,5 @@
-use std::process::Command;
+use std::collections::hash_map::DefaultHasher;
+use std::{hash::Hasher, process::Command};
 
 fn main() {
     // If any TS changes, re-run the build script
@@ -7,6 +8,8 @@ fn main() {
     println!("cargo:rerun-if-changed=src/ts/serialize.ts");
     println!("cargo:rerun-if-changed=src/ts/set_attribute.ts");
     println!("cargo:rerun-if-changed=src/ts/common.ts");
+    println!("cargo:rerun-if-changed=src/ts/eval.ts");
+    println!("cargo:rerun-if-changed=src/ts/native_eval.ts");
 
     // Compute the hash of the ts files
     let hash = hash_ts_files();
@@ -22,34 +25,30 @@ fn main() {
     gen_bindings("common", "common");
     gen_bindings("native", "native");
     gen_bindings("core", "core");
+    gen_bindings("eval", "eval");
+    gen_bindings("native_eval", "native_eval");
 
     std::fs::write("src/js/hash.txt", hash.to_string()).unwrap();
 }
 
 /// Hashes the contents of a directory
-fn hash_ts_files() -> u128 {
-    let mut out = 0;
-
+fn hash_ts_files() -> u64 {
     let files = [
         include_str!("src/ts/common.ts"),
         include_str!("src/ts/native.ts"),
         include_str!("src/ts/core.ts"),
+        include_str!("src/ts/eval.ts"),
+        include_str!("src/ts/native_eval.ts"),
     ];
 
-    // Let's make the dumbest hasher by summing the bytes of the files
-    // The location is multiplied by the byte value to make sure that the order of the bytes matters
-    let mut idx = 0;
+    let mut hash = DefaultHasher::new();
     for file in files {
         // windows + git does a weird thing with line endings, so we need to normalize them
         for line in file.lines() {
-            idx += 1;
-            for byte in line.bytes() {
-                idx += 1;
-                out += (byte as u128) * (idx as u128);
-            }
+            hash.write(line.as_bytes());
         }
     }
-    out
+    hash.finish()
 }
 
 // okay...... so tsc might fail if the user doesn't have it installed

packages/interpreter/src/eval.rs

@@ -0,0 +1,49 @@
+/// Code for the Dioxus channel used to communicate between the dioxus and javascript code
+pub const NATIVE_EVAL_JS: &str = include_str!("./js/native_eval.js");
+
+#[cfg(feature = "webonly")]
+#[wasm_bindgen::prelude::wasm_bindgen]
+pub struct JSOwner {
+    _owner: Box<dyn std::any::Any>,
+}
+
+#[cfg(feature = "webonly")]
+impl JSOwner {
+    pub fn new(owner: impl std::any::Any) -> Self {
+        Self {
+            _owner: Box::new(owner),
+        }
+    }
+}
+
+#[cfg(feature = "webonly")]
+#[wasm_bindgen::prelude::wasm_bindgen(module = "/src/js/eval.js")]
+extern "C" {
+    pub type WebDioxusChannel;
+
+    #[wasm_bindgen(constructor)]
+    pub fn new(owner: JSOwner) -> WebDioxusChannel;
+
+    #[wasm_bindgen(method, js_name = "rustSend")]
+    pub fn rust_send(this: &WebDioxusChannel, value: wasm_bindgen::JsValue);
+
+    #[wasm_bindgen(method, js_name = "rustRecv")]
+    pub async fn rust_recv(this: &WebDioxusChannel) -> wasm_bindgen::JsValue;
+
+    #[wasm_bindgen(method)]
+    pub fn send(this: &WebDioxusChannel, value: wasm_bindgen::JsValue);
+
+    #[wasm_bindgen(method)]
+    pub async fn recv(this: &WebDioxusChannel) -> wasm_bindgen::JsValue;
+
+    #[wasm_bindgen(method)]
+    pub fn weak(this: &WebDioxusChannel) -> WeakDioxusChannel;
+
+    pub type WeakDioxusChannel;
+
+    #[wasm_bindgen(method, js_name = "rustSend")]
+    pub fn rust_send(this: &WeakDioxusChannel, value: wasm_bindgen::JsValue);
+
+    #[wasm_bindgen(method, js_name = "rustRecv")]
+    pub async fn rust_recv(this: &WeakDioxusChannel) -> wasm_bindgen::JsValue;
+}

packages/interpreter/src/js/eval.js

@@ -0,0 +1 @@
+class Channel{pending;waiting;constructor(){this.pending=[],this.waiting=[]}send(data){if(this.waiting.length>0){this.waiting.shift()(data);return}this.pending.push(data)}async recv(){return new Promise((resolve,_reject)=>{if(this.pending.length>0){resolve(this.pending.shift());return}this.waiting.push(resolve)})}}class WeakDioxusChannel{inner;constructor(channel){this.inner=new WeakRef(channel)}rustSend(data){let channel=this.inner.deref();if(channel)channel.rustSend(data)}async rustRecv(){let channel=this.inner.deref();if(channel)return await channel.rustRecv()}}class DioxusChannel{weak(){return new WeakDioxusChannel(this)}}class WebDioxusChannel extends DioxusChannel{js_to_rust;rust_to_js;owner;constructor(owner){super();this.owner=owner,this.js_to_rust=new Channel,this.rust_to_js=new Channel}weak(){return new WeakDioxusChannel(this)}async recv(){return await this.rust_to_js.recv()}send(data){this.js_to_rust.send(data)}rustSend(data){this.rust_to_js.send(data)}async rustRecv(){return await this.js_to_rust.recv()}}export{WebDioxusChannel,WeakDioxusChannel,DioxusChannel,Channel};

packages/interpreter/src/js/hash.txt

@@ -1 +1 @@
-14148301494
+2770005544568683192

packages/interpreter/src/js/native_eval.js

@@ -0,0 +1 @@
+class Channel{pending;waiting;constructor(){this.pending=[],this.waiting=[]}send(data){if(this.waiting.length>0){this.waiting.shift()(data);return}this.pending.push(data)}async recv(){return new Promise((resolve,_reject)=>{if(this.pending.length>0){resolve(this.pending.shift());return}this.waiting.push(resolve)})}}class WeakDioxusChannel{inner;constructor(channel){this.inner=new WeakRef(channel)}rustSend(data){let channel=this.inner.deref();if(channel)channel.rustSend(data)}async rustRecv(){let channel=this.inner.deref();if(channel)return await channel.rustRecv()}}class DioxusChannel{weak(){return new WeakDioxusChannel(this)}}class QueryParams{id;data;constructor(id,method,data){this.id=id,this.data={method,data}}}window.__msg_queues=window.__msg_queues||[];window.finalizationRegistry=window.finalizationRegistry||new FinalizationRegistry(({id})=>{window.ipc.postMessage(JSON.stringify({method:"query",params:new QueryParams(id,"drop")}))});window.getQuery=function(request_id){return window.__msg_queues[request_id]};window.createQuery=function(request_id){return new NativeDioxusChannel(request_id)};class NativeDioxusChannel extends DioxusChannel{rust_to_js;request_id;constructor(request_id){super();this.rust_to_js=new Channel,this.request_id=request_id,window.__msg_queues[request_id]=this.weak(),window.finalizationRegistry.register(this,{id:request_id})}async recv(){return await this.rust_to_js.recv()}send(data){window.ipc.postMessage(JSON.stringify({method:"query",params:new QueryParams(this.request_id,"send",data)}))}rustSend(data){this.rust_to_js.send(data)}async rustRecv(){}}export{NativeDioxusChannel};

packages/interpreter/src/lib.rs

@@ -20,6 +20,9 @@ pub mod unified_bindings;
 #[cfg(feature = "sledgehammer")]
 pub use unified_bindings::*;
 
+#[cfg(feature = "eval")]
+pub mod eval;
+
 // Common bindings for minimal usage.
 #[cfg(all(feature = "minimal_bindings", feature = "webonly"))]
 pub mod minimal_bindings {

packages/interpreter/src/ts/eval.ts

@@ -0,0 +1,108 @@
+// Handle communication between rust and evaluating javascript
+
+export class Channel {
+  pending: any[];
+  waiting: ((data: any) => void)[];
+
+  constructor() {
+    this.pending = [];
+    this.waiting = [];
+  }
+
+  send(data: any) {
+    // If there's a waiting callback, call it
+    if (this.waiting.length > 0) {
+      this.waiting.shift()(data);
+      return;
+    }
+    // Otherwise queue the data
+    this.pending.push(data);
+  }
+
+  async recv(): Promise<any> {
+    return new Promise((resolve, _reject) => {
+      // If data already exists, resolve immediately
+      if (this.pending.length > 0) {
+        resolve(this.pending.shift());
+        return;
+      }
+      // Otherwise queue the resolve callback
+      this.waiting.push(resolve);
+    });
+  }
+}
+
+export class WeakDioxusChannel {
+  inner: WeakRef<DioxusChannel>;
+
+  constructor(channel: DioxusChannel) {
+    this.inner = new WeakRef(channel);
+  }
+
+  // Send data from rust to javascript
+  rustSend(data: any) {
+    let channel = this.inner.deref();
+    if (channel) {
+      channel.rustSend(data);
+    }
+  }
+
+  // Receive data sent from javascript in rust
+  async rustRecv(): Promise<any> {
+    let channel = this.inner.deref();
+    if (channel) {
+      return await channel.rustRecv();
+    }
+  }
+}
+
+export abstract class DioxusChannel {
+  // Return a weak reference to this channel
+  weak(): WeakDioxusChannel {
+    return new WeakDioxusChannel(this);
+  }
+
+  // Send data from rust to javascript
+  abstract rustSend(data: any): void;
+
+  // Receive data sent from javascript in rust
+  abstract rustRecv(): Promise<any>;
+}
+
+export class WebDioxusChannel extends DioxusChannel {
+  js_to_rust: Channel;
+  rust_to_js: Channel;
+  owner: any;
+
+  constructor(owner: any) {
+    super();
+    this.owner = owner;
+    this.js_to_rust = new Channel();
+    this.rust_to_js = new Channel();
+  }
+
+  // Return a weak reference to this channel
+  weak(): WeakDioxusChannel {
+    return new WeakDioxusChannel(this);
+  }
+
+  // Receive message from Rust
+  async recv() {
+    return await this.rust_to_js.recv();
+  }
+
+  // Send message to rust.
+  send(data: any) {
+    this.js_to_rust.send(data);
+  }
+
+  // Send data from rust to javascript
+  rustSend(data: any) {
+    this.rust_to_js.send(data);
+  }
+
+  // Receive data sent from javascript in rust
+  async rustRecv(): Promise<any> {
+    return await this.js_to_rust.recv();
+  }
+}

packages/interpreter/src/ts/native_eval.ts

@@ -0,0 +1,86 @@
+import { Channel, DioxusChannel, WeakDioxusChannel } from "./eval";
+
+// In dioxus desktop, eval needs to use the window object to store global state because we evaluate separate snippets of javascript in the browser
+declare global {
+  interface Window {
+    __msg_queues: WeakDioxusChannel[];
+    finalizationRegistry: FinalizationRegistry<{ id: number }>;
+
+    getQuery(request_id: number): WeakDioxusChannel;
+
+    createQuery(request_id: number): NativeDioxusChannel;
+  }
+}
+
+// A message that can be sent to the desktop renderer about a query
+class QueryParams {
+  id: number;
+  data: { method: "drop" | "send"; data?: any };
+
+  constructor(id: number, method: "drop" | "send", data?: any) {
+    this.id = id;
+    this.data = { method, data };
+  }
+}
+
+window.__msg_queues = window.__msg_queues || [];
+// In dioxus desktop, eval is copy so we cannot run a drop handler. Instead, the drop handler is run after the channel is garbage collected in the javascript side
+window.finalizationRegistry =
+  window.finalizationRegistry ||
+  new FinalizationRegistry(({ id }) => {
+    // @ts-ignore - wry gives us this
+    window.ipc.postMessage(
+      JSON.stringify({
+        method: "query",
+        params: new QueryParams(id, "drop"),
+      })
+    );
+  });
+
+// Get a query from the global state
+window.getQuery = function (request_id: number): WeakDioxusChannel {
+  return window.__msg_queues[request_id];
+};
+
+// Create a new query (and insert it into the global state)
+window.createQuery = function (request_id: number): NativeDioxusChannel {
+  return new NativeDioxusChannel(request_id);
+};
+
+export class NativeDioxusChannel extends DioxusChannel {
+  rust_to_js: Channel;
+  request_id: number;
+
+  constructor(request_id: number) {
+    super();
+    this.rust_to_js = new Channel();
+    this.request_id = request_id;
+
+    window.__msg_queues[request_id] = this.weak();
+    window.finalizationRegistry.register(this, { id: request_id });
+  }
+
+  // Receive message from Rust
+  async recv() {
+    return await this.rust_to_js.recv();
+  }
+
+  // Send message to rust.
+  send(data: any) {
+    // @ts-ignore - wry gives us this
+    window.ipc.postMessage(
+      JSON.stringify({
+        method: "query",
+        params: new QueryParams(this.request_id, "send", data),
+      })
+    );
+  }
+
+  // Send data from rust to javascript
+  rustSend(data: any) {
+    this.rust_to_js.send(data);
+  }
+
+  // Receive data sent from javascript in rust. This is a no-op in the native interpreter because the rust code runs remotely
+  async rustRecv(): Promise<any> {}
+}

packages/interpreter/tsconfig.json

@@ -5,7 +5,8 @@
             "ES2015",
             "DOM",
             "dom",
-            "dom.iterable"
+            "dom.iterable",
+            "ESNext"
         ],
         "noImplicitAny": true,
         "removeComments": true,