chore: enable running stress on on self-hosted runner (#1053)

* chore: run stress test on selfhost runner * chore: Update stress_test.yml * chore: update env * chore: skip stress_test when running integration test
2025-04-19 03:24:42 -04:00 · 2024-12-11 09:43:47 +08:00 · 2024-12-11 09:43:47 +08:00 · 14b2f3c985
commit 14b2f3c985
parent b2f8a95ec4
8 changed files with 258 additions and 30 deletions
--- a/.github/workflows/integration_test.yml
+++ b/.github/workflows/integration_test.yml
@ -130,7 +130,7 @@ jobs:
      - name: Run Tests
        run: |
          echo "Running tests for ${{ matrix.test_service }} with flags: ${{ matrix.test_cmd }}"
-          RUST_LOG="info" DISABLE_CI_TEST_LOG="true" cargo test ${{ matrix.test_cmd }}
+          RUST_LOG="info" DISABLE_CI_TEST_LOG="true" cargo test ${{ matrix.test_cmd }} -- --skip stress_test

      - name: Docker Logs
        if: always()
--- a/.github/workflows/stress_test.yml
+++ b/.github/workflows/stress_test.yml
@ -0,0 +1,49 @@
+name: AppFlowy-Cloud Stress Test
+
+on: [ pull_request ]
+
+concurrency:
+  group: stress-test-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: false
+
+env:
+  POSTGRES_HOST: localhost
+  REDIS_HOST: localhost
+  MINIO_HOST: localhost
+  SQLX_OFFLINE: true
+  RUST_TOOLCHAIN: "1.78"
+
+jobs:
+  test:
+    name: Collab Stress Tests
+    runs-on: self-hosted-appflowy3
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+
+      - name: Install Rust Toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Copy and Rename deploy.env to .env
+        run: cp deploy.env .env
+
+      - name: Replace Values in .env
+        run: |
+          sed -i '' 's|RUST_LOG=.*|RUST_LOG=debug|' .env
+          sed -i '' 's|API_EXTERNAL_URL=.*|API_EXTERNAL_URL=http://localhost:9999|' .env
+          sed -i '' 's|APPFLOWY_GOTRUE_BASE_URL=.*|APPFLOWY_GOTRUE_BASE_URL=http://localhost:9999|' .env
+        shell: bash
+
+      - name: Start Docker Compose Services
+        run: |
+          docker compose -f docker-compose-stress-test.yml up -d 
+          docker ps -a
+
+      - name: Install Prerequisites
+        run: |
+          brew install protobuf
+
+      - name: Run Server and Test
+        run: |
+          cargo run --package xtask -- --stress-test
--- a/Cargo.lock
+++ b/Cargo.lock
@ -3342,9 +3342,9 @@ dependencies = [

 [[package]]
 name = "futures"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0"
+checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
 dependencies = [
 "futures-channel",
 "futures-core",
@ -3373,9 +3373,9 @@ checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"

 [[package]]
 name = "futures-executor"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d"
+checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
 dependencies = [
 "futures-core",
 "futures-task",
@ -8399,6 +8399,7 @@ name = "xtask"
 version = "0.1.0"
 dependencies = [
 "anyhow",
+ "futures",
 "tokio",
 ]

--- a/deploy.env
+++ b/deploy.env
@ -4,7 +4,7 @@
 # PostgreSQL Settings
 POSTGRES_HOST=postgres
 POSTGRES_USER=postgres
-POSTGRES_PASSWORD=changepassword
+POSTGRES_PASSWORD=password
 POSTGRES_PORT=5432
 POSTGRES_DB=postgres

@ -15,6 +15,10 @@ SUPABASE_PASSWORD=root
 REDIS_HOST=redis
 REDIS_PORT=6379

+# Minio Host
+MINIO_HOST=minio
+MINIO_PORT=9000
+
 # AppFlowy Cloud
 ## URL that connects to the gotrue docker container
 APPFLOWY_GOTRUE_BASE_URL=http://gotrue:9999
@ -66,11 +70,12 @@ GOTRUE_ADMIN_PASSWORD=password
 # If you are using a different domain, you need to change the redirect_uri in the OAuth2 configuration
 # Make sure that this domain is accessible to the user
 # Make sure no endswith /
+# Replace with your host name instead of localhost
 API_EXTERNAL_URL=http://your-host

 # In docker environment, `postgres` is the hostname of the postgres service
 # GoTrue connect to postgres using this url
-GOTRUE_DATABASE_URL=postgres://supabase_auth_admin:${SUPABASE_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
+GOTRUE_DATABASE_URL=postgres://supabase_auth_admin:${SUPABASE_PASSWORD}@postgres:${POSTGRES_PORT}/${POSTGRES_DB}

 # Refer to this for details: https://github.com/AppFlowy-IO/AppFlowy-Cloud/blob/main/doc/AUTHENTICATION.md
 # Google OAuth2
@ -102,7 +107,7 @@ APPFLOWY_S3_CREATE_BUCKET=true
 # By default, Minio is used as the default file storage which uses host's file system.
 # Keep this as true if you are using other S3 compatible storage provider other than AWS.
 APPFLOWY_S3_USE_MINIO=true
-APPFLOWY_S3_MINIO_URL=http://minio:9000 # change this if you are using a different address for minio
+APPFLOWY_S3_MINIO_URL=http://${MINIO_HOST}:${MINIO_PORT} # change this if you are using a different address for minio
 APPFLOWY_S3_ACCESS_KEY=minioadmin
 APPFLOWY_S3_SECRET_KEY=minioadmin
 APPFLOWY_S3_BUCKET=appflowy
--- a/docker-compose-stress-test.yml
+++ b/docker-compose-stress-test.yml
@ -0,0 +1,97 @@
+services:
+  nginx:
+    restart: on-failure
+    image: nginx
+    ports:
+      - 80:80   # Disable this if you are using TLS
+      - 443:443
+    volumes:
+      - ./nginx/nginx.conf:/etc/nginx/nginx.conf
+      - ./nginx/ssl/certificate.crt:/etc/nginx/ssl/certificate.crt
+      - ./nginx/ssl/private_key.key:/etc/nginx/ssl/private_key.key
+  minio:
+    restart: on-failure
+    image: minio/minio
+    ports:
+      - 9000:9000
+      - 9001:9001
+    environment:
+      - MINIO_BROWSER_REDIRECT_URL=http://localhost:9001
+    command: server /data --console-address ":9001"
+
+  postgres:
+    restart: on-failure
+    image: pgvector/pgvector:pg16
+    environment:
+      - POSTGRES_USER=${POSTGRES_USER:-postgres}
+      - POSTGRES_DB=${POSTGRES_DB:-postgres}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-password}
+      - POSTGRES_HOST=${POSTGRES_HOST:-postgres}
+      - SUPABASE_USER=${SUPABASE_USER:-supabase_auth_admin}
+      - SUPABASE_PASSWORD=${SUPABASE_PASSWORD:-root}
+    ports:
+      - 5432:5432
+    volumes:
+      - ./migrations/before:/docker-entrypoint-initdb.d
+      # comment out the following line if you want to persist data when restarting docker
+      #- postgres_data:/var/lib/postgresql/data
+
+  redis:
+    restart: on-failure
+    image: redis
+    ports:
+      - 6379:6379
+
+  gotrue:
+    restart: on-failure
+    image: supabase/gotrue:v2.159.1
+    depends_on:
+      - postgres
+    environment:
+      # Gotrue config: https://github.com/supabase/gotrue/blob/master/example.env
+      - GOTRUE_SITE_URL=appflowy-flutter://                           # redirected to AppFlowy application
+      - URI_ALLOW_LIST=*                                              # adjust restrict if necessary
+      - GOTRUE_JWT_SECRET=${GOTRUE_JWT_SECRET}                        # authentication secret
+      - GOTRUE_JWT_EXP=${GOTRUE_JWT_EXP}
+      - GOTRUE_DB_DRIVER=postgres
+      - API_EXTERNAL_URL=${API_EXTERNAL_URL}
+      - DATABASE_URL=${GOTRUE_DATABASE_URL}
+      - PORT=9999
+      - GOTRUE_MAILER_URLPATHS_CONFIRMATION=/verify
+      - GOTRUE_SMTP_HOST=${GOTRUE_SMTP_HOST}                              # e.g. smtp.gmail.com
+      - GOTRUE_SMTP_PORT=${GOTRUE_SMTP_PORT}                              # e.g. 465
+      - GOTRUE_SMTP_USER=${GOTRUE_SMTP_USER}                              # email sender, e.g. noreply@appflowy.io
+      - GOTRUE_SMTP_PASS=${GOTRUE_SMTP_PASS}                              # email password
+      - GOTRUE_SMTP_ADMIN_EMAIL=${GOTRUE_SMTP_ADMIN_EMAIL}                # email with admin privileges e.g. internal@appflowy.io
+      - GOTRUE_SMTP_MAX_FREQUENCY=${GOTRUE_SMTP_MAX_FREQUENCY:-1ns}       # set to 1ns for running tests
+      - GOTRUE_RATE_LIMIT_EMAIL_SENT=${GOTRUE_RATE_LIMIT_EMAIL_SENT:-100} # number of email sendable per minute
+      - GOTRUE_MAILER_AUTOCONFIRM=${GOTRUE_MAILER_AUTOCONFIRM:-false}     # change this to true to skip email confirmation
+      # Google OAuth config
+      - GOTRUE_EXTERNAL_GOOGLE_ENABLED=${GOTRUE_EXTERNAL_GOOGLE_ENABLED}
+      - GOTRUE_EXTERNAL_GOOGLE_CLIENT_ID=${GOTRUE_EXTERNAL_GOOGLE_CLIENT_ID}
+      - GOTRUE_EXTERNAL_GOOGLE_SECRET=${GOTRUE_EXTERNAL_GOOGLE_SECRET}
+      - GOTRUE_EXTERNAL_GOOGLE_REDIRECT_URI=${GOTRUE_EXTERNAL_GOOGLE_REDIRECT_URI}
+      # Apple OAuth config
+      - GOTRUE_EXTERNAL_APPLE_ENABLED=${GOTRUE_EXTERNAL_APPLE_ENABLED}
+      - GOTRUE_EXTERNAL_APPLE_CLIENT_ID=${GOTRUE_EXTERNAL_APPLE_CLIENT_ID}
+      - GOTRUE_EXTERNAL_APPLE_SECRET=${GOTRUE_EXTERNAL_APPLE_SECRET}
+      - GOTRUE_EXTERNAL_APPLE_REDIRECT_URI=${GOTRUE_EXTERNAL_APPLE_REDIRECT_URI}
+      # GITHUB OAuth config
+      - GOTRUE_EXTERNAL_GITHUB_ENABLED=${GOTRUE_EXTERNAL_GITHUB_ENABLED}
+      - GOTRUE_EXTERNAL_GITHUB_CLIENT_ID=${GOTRUE_EXTERNAL_GITHUB_CLIENT_ID}
+      - GOTRUE_EXTERNAL_GITHUB_SECRET=${GOTRUE_EXTERNAL_GITHUB_SECRET}
+      - GOTRUE_EXTERNAL_GITHUB_REDIRECT_URI=${GOTRUE_EXTERNAL_GITHUB_REDIRECT_URI}
+      # Discord OAuth config
+      - GOTRUE_EXTERNAL_DISCORD_ENABLED=${GOTRUE_EXTERNAL_DISCORD_ENABLED}
+      - GOTRUE_EXTERNAL_DISCORD_CLIENT_ID=${GOTRUE_EXTERNAL_DISCORD_CLIENT_ID}
+      - GOTRUE_EXTERNAL_DISCORD_SECRET=${GOTRUE_EXTERNAL_DISCORD_SECRET}
+      - GOTRUE_EXTERNAL_DISCORD_REDIRECT_URI=${GOTRUE_EXTERNAL_DISCORD_REDIRECT_URI}
+      # Prometheus Metrics
+      - GOTRUE_METRICS_ENABLED=true
+      - GOTRUE_METRICS_EXPORTER=prometheus
+      - GOTRUE_MAILER_TEMPLATES_CONFIRMATION=${GOTRUE_MAILER_TEMPLATES_CONFIRMATION}
+    ports:
+      - 9999:9999
+
+volumes:
+  postgres_data:
--- a/tests/collab/stress_test.rs
+++ b/tests/collab/stress_test.rs
@ -11,7 +11,7 @@ use client_api_test::{assert_server_collab, TestClient};
 use database_entity::dto::AFRole;

 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-async fn run_multiple_text_edits() {
+async fn stress_test_run_multiple_text_edits() {
  const READER_COUNT: usize = 1;
  let test_scenario = Arc::new(TestScenario::open(
    "./tests/collab/asset/automerge-paper.json.gz",
--- a/xtask/Cargo.toml
+++ b/xtask/Cargo.toml
@ -7,4 +7,5 @@ edition = "2021"

 [dependencies]
 anyhow = "1.0"
-tokio = { version = "1", features = ["full"] }
+tokio = { version = "1", features = ["full"] }
+futures = "0.3.31"
--- a/xtask/src/main.rs
+++ b/xtask/src/main.rs
@ -1,48 +1,112 @@
 use anyhow::{anyhow, Context, Result};
+use std::process::Stdio;
 use tokio::process::Command;
 use tokio::select;
+use tokio::time::{sleep, Duration};

-/// Using 'cargo run --package xtask' to run servers in parallel.
-/// 1. AppFlowy Cloud
-/// 2. AppFlowy History
-/// 3. AppFlowy Indexer
+/// Run servers:
+/// cargo run --package xtask
 ///
-/// Before running this command, make sure the other dependencies servers are running. For example,
-/// Redis, Postgres, etc.
+/// Run servers and stress tests:
+/// cargo run --package xtask -- --stress-test
+///
+/// Note: test start with 'stress_test' will be run as stress tests
 #[tokio::main]
 async fn main() -> Result<()> {
+  let is_stress_test = std::env::args().any(|arg| arg == "--stress-test");
+
  let appflowy_cloud_bin_name = "appflowy_cloud";
-  let worker = "appflowy_worker";
+  let worker_bin_name = "appflowy_worker";

+  // Step 1: Kill existing processes
  kill_existing_process(appflowy_cloud_bin_name).await?;
-  kill_existing_process(worker).await?;
+  kill_existing_process(worker_bin_name).await?;

-  let mut appflowy_cloud_cmd = Command::new("cargo")
-    .args(["run", "--features", "history"])
-    .spawn()
-    .context("Failed to start AppFlowy-Cloud process")?;
+  // Step 2: Start servers sequentially
+  println!("Starting {} server...", appflowy_cloud_bin_name);
+  let mut appflowy_cloud_cmd = spawn_server(
+    "cargo",
+    &["run", "--features", "history"],
+    appflowy_cloud_bin_name,
+    is_stress_test,
+  )?;
+  wait_for_readiness(appflowy_cloud_bin_name).await?;

-  let mut appflowy_worker_cmd = Command::new("cargo")
-    .args([
+  println!("Starting {} server...", worker_bin_name);
+  let mut appflowy_worker_cmd = spawn_server(
+    "cargo",
+    &[
      "run",
      "--manifest-path",
      "./services/appflowy-worker/Cargo.toml",
-    ])
-    .spawn()
-    .context("Failed to start AppFlowy-Worker process")?;
+    ],
+    worker_bin_name,
+    is_stress_test,
+  )?;
+  wait_for_readiness(worker_bin_name).await?;

+  println!("All servers are up and running.");
+
+  // Step 3: Run stress tests if flag is set
+  let stress_test_cmd = if is_stress_test {
+    println!("Running stress tests (tests starting with 'stress_test')...");
+    Some(
+      Command::new("cargo")
+        .args(["test", "stress_test", "--", "--nocapture"])
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit())
+        .spawn()
+        .context("Failed to start stress test process")?,
+    )
+  } else {
+    None
+  };
+
+  // Step 4: Monitor all processes
  select! {
      status = appflowy_cloud_cmd.wait() => {
          handle_process_exit(status?, appflowy_cloud_bin_name)?;
      },
      status = appflowy_worker_cmd.wait() => {
-          handle_process_exit(status?, worker)?;
-      }
+          handle_process_exit(status?, worker_bin_name)?;
+      },
+      status = async {
+          if let Some(mut stress_cmd) = stress_test_cmd {
+              stress_cmd.wait().await
+          } else {
+              futures::future::pending().await
+          }
+      } => {
+          if is_stress_test {
+              handle_process_exit(status?, "cargo test stress_test")?;
+          }
+      },
  }

  Ok(())
 }

+fn spawn_server(
+  command: &str,
+  args: &[&str],
+  name: &str,
+  suppress_output: bool,
+) -> Result<tokio::process::Child> {
+  println!("Spawning {} process...", name);
+  let mut cmd = Command::new(command);
+  cmd.args(args);
+
+  if suppress_output {
+    cmd.stdout(Stdio::null()).stderr(Stdio::null());
+  }
+
+  Ok(
+    cmd
+      .spawn()
+      .context(format!("Failed to start {} process", name))?,
+  )
+}
+
 async fn kill_existing_process(process_identifier: &str) -> Result<()> {
  let _ = Command::new("pkill")
    .arg("-f")
@ -59,6 +123,17 @@ fn handle_process_exit(status: std::process::ExitStatus, process_name: &str) ->
    println!("{} exited normally.", process_name);
    Ok(())
  } else {
-    Err(anyhow!("{} process failed", process_name))
+    Err(anyhow!(
+      "{} process failed with code {}",
+      process_name,
+      status
+    ))
  }
 }
+
+async fn wait_for_readiness(process_name: &str) -> Result<()> {
+  println!("Waiting for {} to be ready...", process_name);
+  sleep(Duration::from_secs(3)).await;
+  println!("{} is ready.", process_name);
+  Ok(())
+}