Fix ssg race condition (#3521)

* Fix server crash logging and opening fullstack apps

* Fix race condition during SSG

* move route prerendering into a separate stage

* Fix static site generation with dx serve

* only proxy the fullstack server if ssg is disabled

* test nested-suspense with ssg in playwright

packages/cli/Cargo.toml

@@ -67,9 +67,8 @@ dunce = { workspace = true }
 dirs = { workspace = true }
 reqwest = { workspace = true, features = [
     "rustls-tls",
-    "stream",
     "trust-dns",
-    "blocking",
+    "json"
 ] }
 tower = { workspace = true }
 once_cell = "1.19.0"

packages/cli/src/build/bundle.rs

@@ -1,3 +1,4 @@
+use super::prerender::pre_render_static_routes;
 use super::templates::InfoPlistData;
 use crate::wasm_bindgen::WasmBindgenBuilder;
 use crate::{BuildRequest, Platform};
@@ -284,6 +285,7 @@ impl AppBundle {
             .context("Failed to write assets")?;
         bundle.write_metadata().await?;
         bundle.optimize().await?;
+        bundle.pre_render_ssg_routes().await?;
         bundle
             .assemble()
             .await
@@ -558,11 +560,6 @@ impl AppBundle {
                 })
                 .await
                 .unwrap()?;
-
-                // Run SSG and cache static routes
-                if self.build.build.ssg {
-                    self.run_ssg().await?;
-                }
             }
             Platform::MacOS => {}
             Platform::Windows => {}
@@ -686,76 +683,18 @@ impl AppBundle {
         Ok(())
     }
 
-    async fn run_ssg(&self) -> anyhow::Result<()> {
-        use futures_util::stream::FuturesUnordered;
-        use futures_util::StreamExt;
-        use tokio::process::Command;
-
-        let fullstack_address = dioxus_cli_config::fullstack_address_or_localhost();
-        let address = fullstack_address.ip().to_string();
-        let port = fullstack_address.port().to_string();
-
-        tracing::info!("Running SSG");
-
-        // Run the server executable
-        let _child = Command::new(
-            self.server_exe()
+    async fn pre_render_ssg_routes(&self) -> Result<()> {
+        // Run SSG and cache static routes
+        if !self.build.build.ssg {
+            return Ok(());
+        }
+        self.build.status_prerendering_routes();
+        pre_render_static_routes(
+            &self
+                .server_exe()
                 .context("Failed to find server executable")?,
         )
-        .env(dioxus_cli_config::SERVER_PORT_ENV, port.clone())
-        .env(dioxus_cli_config::SERVER_IP_ENV, address.clone())
-        .stdout(std::process::Stdio::piped())
-        .stderr(std::process::Stdio::piped())
-        .kill_on_drop(true)
-        .spawn()?;
-
-        // Wait a second for the server to start
-        tokio::time::sleep(std::time::Duration::from_secs(1)).await;
-
-        // Get the routes from the `/static_routes` endpoint
-        let mut routes = reqwest::Client::builder()
-            .build()?
-            .post(format!("http://{address}:{port}/api/static_routes"))
-            .send()
-            .await
-            .context("Failed to get static routes from server")?
-            .text()
-            .await
-            .map(|raw| serde_json::from_str::<Vec<String>>(&raw).unwrap())
-            .inspect(|text| tracing::debug!("Got static routes: {text:?}"))
-            .context("Failed to parse static routes from server")?
-            .into_iter()
-            .map(|line| {
-                let port = port.clone();
-                let address = address.clone();
-                async move {
-                    tracing::info!("SSG: {line}");
-                    reqwest::Client::builder()
-                        .build()?
-                        .get(format!("http://{address}:{port}{line}"))
-                        .header("Accept", "text/html")
-                        .send()
-                        .await
-                }
-            })
-            .collect::<FuturesUnordered<_>>();
-
-        while let Some(route) = routes.next().await {
-            match route {
-                Ok(route) => tracing::debug!("ssg success: {route:?}"),
-                Err(err) => tracing::error!("ssg error: {err:?}"),
-            }
-        }
-
-        // Wait a second for the cache to be written by the server
-        tracing::info!("Waiting a moment for isrg to propagate...");
-
-        tokio::time::sleep(std::time::Duration::from_secs(10)).await;
-
-        tracing::info!("SSG complete");
-
-        drop(_child);
-
+        .await?;
         Ok(())
     }
 

packages/cli/src/build/mod.rs

@@ -7,6 +7,7 @@
 
 mod builder;
 mod bundle;
+mod prerender;
 mod progress;
 mod request;
 mod templates;

packages/cli/src/build/prerender.rs

@@ -0,0 +1,116 @@
+use anyhow::Context;
+use dioxus_cli_config::{server_ip, server_port};
+use futures_util::stream::FuturesUnordered;
+use futures_util::StreamExt;
+use std::{
+    net::{IpAddr, Ipv4Addr, SocketAddr},
+    path::Path,
+    time::Duration,
+};
+use tokio::process::Command;
+
+pub(crate) async fn pre_render_static_routes(server_exe: &Path) -> anyhow::Result<()> {
+    // Use the address passed in through environment variables or default to localhost:9999. We need
+    // to default to a value that is different than the CLI default address to avoid conflicts
+    let ip = server_ip().unwrap_or_else(|| IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)));
+    let port = server_port().unwrap_or(9999);
+    let fullstack_address = SocketAddr::new(ip, port);
+    let address = fullstack_address.ip().to_string();
+    let port = fullstack_address.port().to_string();
+    // Borrow port and address so we can easily moe them into multiple tasks below
+    let address = &address;
+    let port = &port;
+
+    tracing::info!("Running SSG at http://{address}:{port}");
+
+    // Run the server executable
+    let _child = Command::new(server_exe)
+        .env(dioxus_cli_config::SERVER_PORT_ENV, port)
+        .env(dioxus_cli_config::SERVER_IP_ENV, address)
+        .stdout(std::process::Stdio::piped())
+        .stderr(std::process::Stdio::piped())
+        .kill_on_drop(true)
+        .spawn()?;
+
+    let reqwest_client = reqwest::Client::new();
+    // Borrow reqwest_client so we only move the reference into the futures
+    let reqwest_client = &reqwest_client;
+
+    // Get the routes from the `/static_routes` endpoint
+    let mut routes = None;
+
+    // The server may take a few seconds to start up. Try fetching the route up to 5 times with a one second delay
+    const RETRY_ATTEMPTS: usize = 5;
+    for i in 0..=RETRY_ATTEMPTS {
+        let request = reqwest_client
+            .post(format!("http://{address}:{port}/api/static_routes"))
+            .send()
+            .await;
+        match request {
+            Ok(request) => {
+                routes = Some(request
+                    .json::<Vec<String>>()
+                    .await
+                    .inspect(|text| tracing::debug!("Got static routes: {text:?}"))
+                    .context("Failed to parse static routes from the server. Make sure your server function returns Vec<String> with the (default) json encoding")?);
+                break;
+            }
+            Err(err) => {
+                // If the request fails, try  up to 5 times with a one second delay
+                // If it fails 5 times, return the error
+                if i == RETRY_ATTEMPTS {
+                    return Err(err).context("Failed to get static routes from server. Make sure you have a server function at the `/api/static_routes` endpoint that returns Vec<String> of static routes.");
+                }
+                tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+            }
+        }
+    }
+
+    let routes = routes.expect(
+        "static routes should exist or an error should have been returned on the last attempt",
+    );
+
+    // Create a pool of futures that cache each route
+    let mut resolved_routes = routes
+        .into_iter()
+        .map(|route| async move {
+            tracing::info!("Rendering {route} for SSG");
+            // For each route, ping the server to force it to cache the response for ssg
+            let request = reqwest_client
+                .get(format!("http://{address}:{port}{route}"))
+                .header("Accept", "text/html")
+                .send()
+                .await?;
+            // If it takes longer than 30 seconds to resolve the route, log a warning
+            let warning_task = tokio::spawn({
+                let route = route.clone();
+                async move {
+                    tokio::time::sleep(Duration::from_secs(30)).await;
+                    tracing::warn!("Route {route} has been rendering for 30 seconds");
+                }
+            });
+            // Wait for the streaming response to completely finish before continuing. We don't use the html it returns directly
+            // because it may contain artifacts of intermediate streaming steps while the page is loading. The SSG app should write
+            // the final clean HTML to the disk automatically after the request completes.
+            let _html = request.text().await?;
+
+            // Cancel the warning task if it hasn't already run
+            warning_task.abort();
+
+            Ok::<_, reqwest::Error>(route)
+        })
+        .collect::<FuturesUnordered<_>>();
+
+    while let Some(route) = resolved_routes.next().await {
+        match route {
+            Ok(route) => tracing::debug!("ssg success: {route:?}"),
+            Err(err) => tracing::error!("ssg error: {err:?}"),
+        }
+    }
+
+    tracing::info!("SSG complete");
+
+    drop(_child);
+
+    Ok(())
+}

packages/cli/src/build/progress.rs

@@ -90,6 +90,12 @@ impl BuildRequest {
         });
     }
 
+    pub(crate) fn status_prerendering_routes(&self) {
+        _ = self.progress.unbounded_send(BuildUpdate::Progress {
+            stage: BuildStage::PrerenderingRoutes {},
+        });
+    }
+
     pub(crate) fn status_installing_tooling(&self) {
         _ = self.progress.unbounded_send(BuildUpdate::Progress {
             stage: BuildStage::InstallingTooling {},

packages/cli/src/cli/serve.rs

@@ -103,7 +103,8 @@ impl ServeArgs {
     pub(crate) fn should_proxy_build(&self) -> bool {
         match self.build_arguments.platform() {
             Platform::Server => true,
-            _ => self.build_arguments.fullstack,
+            // During SSG, just serve the static files instead of running the server
+            _ => self.build_arguments.fullstack && !self.build_arguments.ssg,
         }
     }
 }

packages/cli/src/serve/handle.rs

@@ -60,13 +60,9 @@ impl AppHandle {
     pub(crate) async fn open(
         &mut self,
         devserver_ip: SocketAddr,
-        fullstack_address: Option<SocketAddr>,
+        start_fullstack_on_address: Option<SocketAddr>,
         open_browser: bool,
     ) -> Result<()> {
-        if let Some(addr) = fullstack_address {
-            tracing::debug!("Proxying fullstack server from port {:?}", addr);
-        }
-
         // Set the env vars that the clients will expect
         // These need to be stable within a release version (ie 0.6.0)
         let mut envs = vec![
@@ -89,13 +85,12 @@ impl AppHandle {
             envs.push((dioxus_cli_config::ASSET_ROOT_ENV, base_path.clone()));
         }
 
-        if let Some(addr) = fullstack_address {
+        // Launch the server if we were given an address to start it on, and the build includes a server. After we
+        // start the server, consume its stdout/stderr.
+        if let (Some(addr), Some(server)) = (start_fullstack_on_address, self.app.server_exe()) {
+            tracing::debug!("Proxying fullstack server from port {:?}", addr);
             envs.push((dioxus_cli_config::SERVER_IP_ENV, addr.ip().to_string()));
             envs.push((dioxus_cli_config::SERVER_PORT_ENV, addr.port().to_string()));
-        }
-
-        // Launch the server if we have one and consume its stdout/stderr
-        if let Some(server) = self.app.server_exe() {
             tracing::debug!("Launching server from path: {server:?}");
             let mut child = Command::new(server)
                 .envs(envs.clone())

packages/cli/src/serve/output.rs

@@ -482,6 +482,7 @@ impl Output {
                 lines.push(krate.as_str().dark_gray())
             }
             BuildStage::OptimizingWasm {} => lines.push("Optimizing wasm".yellow()),
+            BuildStage::PrerenderingRoutes {} => lines.push("Prerendering static routes".yellow()),
             BuildStage::RunningBindgen {} => lines.push("Running wasm-bindgen".yellow()),
             BuildStage::RunningGradle {} => lines.push("Running gradle assemble".yellow()),
             BuildStage::Bundling {} => lines.push("Bundling app".yellow()),

packages/cli/src/serve/runner.rs

@@ -79,7 +79,7 @@ impl AppRunner {
                     },
                     Some(status) = OptionFuture::from(handle.server_child.as_mut().map(|f| f.wait())) => {
                         match status {
-                            Ok(status) => ProcessExited { status, platform: Platform::Server },
+                            Ok(status) => ProcessExited { status, platform },
                             Err(_err) => todo!("handle error in process joining?"),
                         }
                     }

packages/cli/src/serve/server.rs

@@ -383,18 +383,18 @@ fn build_devserver_router(
         // For fullstack, liveview, and server, forward all requests to the inner server
         let address = fullstack_address.unwrap();
         router = router.nest_service("/",super::proxy::proxy_to(
-                format!("http://{address}").parse().unwrap(),
-                true,
-                |error| {
-                    Response::builder()
-                        .status(StatusCode::INTERNAL_SERVER_ERROR)
-                        .body(Body::from(format!(
-                            "Backend connection failed. The backend is likely still starting up. Please try again in a few seconds. Error: {:#?}",
-                            error
-                        )))
-                        .unwrap()
-                },
-            ));
+            format!("http://{address}").parse().unwrap(),
+            true,
+            |error| {
+                Response::builder()
+                    .status(StatusCode::INTERNAL_SERVER_ERROR)
+                    .body(Body::from(format!(
+                        "Backend connection failed. The backend is likely still starting up. Please try again in a few seconds. Error: {:#?}",
+                        error
+                    )))
+                    .unwrap()
+            },
+        ));
     } else {
         // Otherwise, just serve the dir ourselves
         // Route file service to output the .wasm and assets if this is a web build