From 80d46a2e3ca1f0199d72b09f91cf24a2e472eecc Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Mon, 27 May 2024 16:48:13 +0800 Subject: [PATCH] chore: Clean redis update script (#581) * chore: rename folder * chore: add script to remove redis stream * chore: add script to remove redis stream * chore: fix ci --- .github/workflows/client_api_check.yml | 2 +- .github/workflows/rustlint.yml | 2 +- Makefile | 2 +- README.md | 20 +++-- doc/CONTRIBUTING.md | 39 ++++++---- libs/client-api-test/src/client.rs | 2 +- {build => script}/client_api_deps_check.sh | 0 {build => script}/code_gen.sh | 0 {build => script}/init_database.sh | 0 {build => script}/init_gotrue.sh | 0 {build => script}/init_redis.sh | 0 script/redis/remove_redis_stream_range.sh | 87 ++++++++++++++++++++++ script/redis/show_redis_stream_values.sh | 45 +++++++++++ {build => script}/run_ci_server.sh | 0 {build => script}/run_local_server.sh | 2 +- 15 files changed, 176 insertions(+), 25 deletions(-) rename {build => script}/client_api_deps_check.sh (100%) rename {build => script}/code_gen.sh (100%) rename {build => script}/init_database.sh (100%) rename {build => script}/init_gotrue.sh (100%) rename {build => script}/init_redis.sh (100%) create mode 100755 script/redis/remove_redis_stream_range.sh create mode 100755 script/redis/show_redis_stream_values.sh rename {build => script}/run_ci_server.sh (100%) rename {build => script}/run_local_server.sh (98%) diff --git a/.github/workflows/client_api_check.yml b/.github/workflows/client_api_check.yml index d94c6f3b..0e82244a 100644 --- a/.github/workflows/client_api_check.yml +++ b/.github/workflows/client_api_check.yml @@ -35,5 +35,5 @@ jobs: - name: Check ClientAPI Dependencies working-directory: ./libs/client-api - run: bash ../../build/client_api_deps_check.sh + run: bash ../../script/client_api_deps_check.sh diff --git a/.github/workflows/rustlint.yml b/.github/workflows/rustlint.yml index cf7dd7b1..1e2430db 100644 --- a/.github/workflows/rustlint.yml +++ b/.github/workflows/rustlint.yml @@ -44,7 +44,7 @@ jobs: run: cp dev.env .env - name: Code Gen - working-directory: ./build + working-directory: ./script run: ./code_gen.sh - name: Rustfmt diff --git a/Makefile b/Makefile index fbf5c8f5..1b7aae94 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ROOT = "./build" +ROOT = "./script" SEMVER_VERSION=$(shell grep version Cargo.toml | awk -F"\"" '{print $$2}' | head -n 1) .PHONY: run test diff --git a/README.md b/README.md index a2e799b3..c2694417 100644 --- a/README.md +++ b/README.md @@ -21,13 +21,14 @@ AppFlowy Cloud is part of the AppFlowy ecosystem, offering secure user authentic and real-time WebSocket communication for an efficient and collaborative user experience. ## Table of Contents + - [🚀 Deployment](#-deployment) - [💻 Development](#-development) - [🐞 Debugging](#-debugging) - [⚙️ Contributing](#-contributing) - ## 🚀 Deployment + - See [deployment guide](./doc/DEPLOYMENT.md) ## 💻 Development @@ -40,12 +41,14 @@ You'll need to install: - [Docker](https://docs.docker.com/get-docker/) ### Configuration + - copy the configurations from `dev.env` to `.env` - edit the `.env` as required (such as SMTP configurations) ### Run with all dependencies + ```bash -./build/run_local_server.sh +./script/run_local_server.sh ``` This process will execute all the dependencies and start the AppFlowy-Cloud server. Alternatively, @@ -54,16 +57,19 @@ you have the option to run the AppFlowy-Cloud server independently ### Run the AppFlowy-Cloud 1. Run the dependency servers + ```bash docker compose --file docker-compose-dev.yml up -d ``` 2. Install sqlx-cli + ```bash cargo install sqlx-cli ``` 3. Run sqlx migration + ```bash sqlx database create sqlx migrate run @@ -71,6 +77,7 @@ cargo sqlx prepare --workspace ``` 4. Run the server + ```bash cargo run ``` @@ -93,6 +100,7 @@ troubleshoot issues in various components of the AppFlowy cloud server: A web-based administration tool for PostgreSQL. Access it at [PgAdmin](http://localhost:5400) - OR command line: + ```bash export PGPASSWORD=password psql --host=localhost --username=postgres --port=5432 @@ -111,11 +119,13 @@ Redis offers a powerful command line interface for managing your Redis instance. Minio provides a Web UI for easy management of your files and buckets. Access it at [Web UI](http://localhost:9001) ### Portainer -For managing Docker containers, Portainer's Web UI is an excellent tool. Access it at Web UI to easily manage Docker -environments, including container deployment, networking, volume management, and more. Access it at [Web UI](http://localhost:9442) +For managing Docker containers, Portainer's Web UI is an excellent tool. Access it at Web UI to easily manage Docker +environments, including container deployment, networking, volume management, and more. Access it +at [Web UI](http://localhost:9442) ## ⚙️ Contributing Any new contribution is more than welcome in this project! -If you want to know more about the development workflow or want to contribute, please visit our [contributing guidelines](./doc/CONTRIBUTING.md) for detailed instructions! +If you want to know more about the development workflow or want to contribute, please visit +our [contributing guidelines](./doc/CONTRIBUTING.md) for detailed instructions! diff --git a/doc/CONTRIBUTING.md b/doc/CONTRIBUTING.md index 659dec6b..5fb61a53 100644 --- a/doc/CONTRIBUTING.md +++ b/doc/CONTRIBUTING.md @@ -1,40 +1,47 @@ # Contributing -First of all, thank you for contributing to AppFlowy Cloud! The goal of this document is to provide everything you need +First of all, thank you for contributing to AppFlowy Cloud! The goal of this document is to provide everything you need to know in order to contribute to AppFlowy Cloud and its different integrations. - [Assumptions](#assumptions) - [How to Contribute](#how-to-contribute) - [Development Workflow](#development-workflow) - ## Assumptions -1. **You're familiar with [GitHub](https://github.com) and the [Pull Request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests)(PR) workflow.** +1. **You're familiar with [GitHub](https://github.com) and + the [Pull Request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests)( + PR) workflow.** 2. **You know about the [AppFlowy community](https://discord.gg/9Q2xaN37tV"). Please use this for help.** -## How to Contribute +## How to Contribute Contributions are welcome! Here's how you can help improve AppFlowy Cloud: -1. Identify or propose enhancements or fixes by checking [existing issues](https://github.com/AppFlowy-IO/AppFlowy-Cloud/issues) or [creating a new one](https://github.com/AppFlowy-IO/AppFlowy-Cloud/issues/new/choose). -2. [Fork the repository](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) to your own GitHub account. Feel free to discuss your contribution with a maintainer beforehand. -3. [Create a feature or bugfix branch](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-and-deleting-branches-within-your-repository) in your forked repo. -4. Familiarize yourself with the [Development Workflow](#development-workflow) for guidelines on maintaining code quality. +1. Identify or propose enhancements or fixes by + checking [existing issues](https://github.com/AppFlowy-IO/AppFlowy-Cloud/issues) + or [creating a new one](https://github.com/AppFlowy-IO/AppFlowy-Cloud/issues/new/choose). +2. [Fork the repository](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) to your own GitHub + account. Feel free to discuss your contribution with a maintainer beforehand. +3. [Create a feature or bugfix branch](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-and-deleting-branches-within-your-repository) + in your forked repo. +4. Familiarize yourself with the [Development Workflow](#development-workflow) for guidelines on maintaining code + quality. 5. Implement your changes on the new branch. -6. [Open a Pull Request (PR)](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request-from-a-fork) against the `main` branch of the original AppFlowy Cloud repo. Await feedback or approval from the maintainers. +6. [Open a Pull Request (PR)](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request-from-a-fork) + against the `main` branch of the original AppFlowy Cloud repo. Await feedback or approval from the maintainers. +## Development Workflow -## Development Workflow - -Before diving into development, familiarize yourself with the codebase and project standards by reviewing the [Development Guide](./GUIDE.md). +Before diving into development, familiarize yourself with the codebase and project standards by reviewing +the [Development Guide](./GUIDE.md). ### Setting Up the Local Server To start the server on your local machine, run the following script: ```bash -./build/run_local_server.sh +./script/run_local_server.sh ``` ### Testing @@ -51,7 +58,8 @@ For a pull request to be accepted, it must satisfy the following criteria: 1. **Pass All Tests**: Your PR should not break any existing functionality and must pass all the automated tests. -2. **Linting with Clippy**: Your code must adhere to the linting standards enforced by [`clippy`](https://github.com/rust-lang/rust-clippy). You can check for linting issues using: +2. **Linting with Clippy**: Your code must adhere to the linting standards enforced + by [`clippy`](https://github.com/rust-lang/rust-clippy). You can check for linting issues using: ```bash cargo clippy -- -D warnings @@ -64,7 +72,8 @@ For a pull request to be accepted, it must satisfy the following criteria: rustup component add clippy ``` -3. **Code Formatting**: The code must comply with established formatting rules. Use the following commands for formatting and checking your code: +3. **Code Formatting**: The code must comply with established formatting rules. Use the following commands for + formatting and checking your code: To format your code: diff --git a/libs/client-api-test/src/client.rs b/libs/client-api-test/src/client.rs index c49a272c..47054d3d 100644 --- a/libs/client-api-test/src/client.rs +++ b/libs/client-api-test/src/client.rs @@ -38,7 +38,7 @@ fn get_env_var<'default>(key: &str, default: &'default str) -> Cow<'default, str /// Return a client that connects to the local host. It requires to run the server locally. /// ```shell -/// ./build/run_local_server.sh +/// ./script/run_local_server.sh /// ``` pub fn localhost_client() -> Client { let device_id = Uuid::new_v4().to_string(); diff --git a/build/client_api_deps_check.sh b/script/client_api_deps_check.sh similarity index 100% rename from build/client_api_deps_check.sh rename to script/client_api_deps_check.sh diff --git a/build/code_gen.sh b/script/code_gen.sh similarity index 100% rename from build/code_gen.sh rename to script/code_gen.sh diff --git a/build/init_database.sh b/script/init_database.sh similarity index 100% rename from build/init_database.sh rename to script/init_database.sh diff --git a/build/init_gotrue.sh b/script/init_gotrue.sh similarity index 100% rename from build/init_gotrue.sh rename to script/init_gotrue.sh diff --git a/build/init_redis.sh b/script/init_redis.sh similarity index 100% rename from build/init_redis.sh rename to script/init_redis.sh diff --git a/script/redis/remove_redis_stream_range.sh b/script/redis/remove_redis_stream_range.sh new file mode 100755 index 00000000..8e6c5050 --- /dev/null +++ b/script/redis/remove_redis_stream_range.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +# delete all Redis stream entries matching the prefix that are older than the given date range +# Usage: ./remove_redis_stream_range.sh stream_prefix 20231001 20231005 [--verbose] + +# Function to convert date string (YYYYMMDD) to Unix timestamp in milliseconds +date_to_timestamp_ms() { + date -j -f "%Y%m%d" "$1" "+%s000" 2>/dev/null || date -d "$1" "+%s000" 2>/dev/null +} + +# Check if the correct number of arguments is provided +if [ "$#" -lt 3 ] || [ "$#" -gt 4 ]; then + echo "Usage: $0 [--verbose]" + exit 1 +fi + +# Check if redis-cli is installed +if ! command -v redis-cli &> /dev/null; then + echo "redis-cli could not be found. Please install Redis CLI." + exit 1 +fi + +# Input stream prefix and date range in YYYYMMDD format +stream_prefix=$1 +start_date=$2 +end_date=$3 +verbose=false + +# Check for verbose flag +if [ "$#" -eq 4 ] && [ "$4" == "--verbose" ]; then + verbose=true +fi + +# Validate input date format +if ! [[ $start_date =~ ^[0-9]{8}$ ]] || ! [[ $end_date =~ ^[0-9]{8}$ ]]; then + echo "Invalid date format. Please use YYYYMMDD." + exit 1 +fi + +# Convert input dates to Unix timestamps in milliseconds +start_timestamp=$(date_to_timestamp_ms $start_date) +end_timestamp=$(date_to_timestamp_ms $end_date) + +if [ $? -ne 0 ] || [ -z "$start_timestamp" ] || [ -z "$end_timestamp" ]; then + echo "Error converting date to timestamp. Please check the input dates." + exit 1 +fi + +# Construct Redis stream IDs +start_id="${start_timestamp}-0" +end_id="${end_timestamp}-0" + +# Initialize cursor for SCAN +cursor=0 + +while :; do + # Scan for keys with the specified prefix + result=$(redis-cli scan $cursor match "${stream_prefix}*") + + # Extract cursor and keys + cursor=$(echo "$result" | head -n 1) + keys=$(echo "$result" | tail -n +2 | tr -d '\r') + + # Loop through the keys and delete entries within the range + for key in $keys; do + if $verbose; then + echo "Query entries from stream: $key in the range: $start_id to $end_id" + fi + + # Fetch entries within the range + entries=$(redis-cli xrange $key $start_id $end_id) + + # Loop through the entries and delete them + while IFS= read -r entry; do + entry_id=$(echo $entry | awk '{print $1}') + if [[ $entry_id =~ ^[0-9]+-[0-9]+$ ]]; then + redis-cli xdel $key $entry_id + if $verbose; then + echo "Deleted entry: $entry_id from stream: $key" + fi + fi + done <<< "$entries" + done + + # Break loop if cursor is 0 + [ "$cursor" -eq 0 ] && break +done diff --git a/script/redis/show_redis_stream_values.sh b/script/redis/show_redis_stream_values.sh new file mode 100755 index 00000000..349a277c --- /dev/null +++ b/script/redis/show_redis_stream_values.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Show all Redis stream values matching the given prefix +# Usage: ./show_redis_stream_values.sh stream_prefix + +# Check if the correct number of arguments is provided +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Check if redis-cli is installed +if ! command -v redis-cli &> /dev/null +then + echo "redis-cli could not be found. Please install Redis CLI." + exit 1 +fi + +# Input stream prefix +stream_prefix=$1 + +# Initialize cursor for SCAN +cursor=0 + +while :; do + # Scan for keys with the specified prefix + result=$(redis-cli scan $cursor match "${stream_prefix}*") + + # Extract cursor and keys + cursor=$(echo "$result" | head -n 1) + keys=$(echo "$result" | tail -n +2) + echo "Found keys: $(echo $keys | wc -w)" + + # Loop through the keys and show entries + for key in $keys; do + echo "Entries for stream key: $key" + redis-cli xrange $key - + | while read -r entry_id fields; do + echo "Entry ID: $entry_id" + echo "Fields: $fields" + done + done + + # Break loop if cursor is 0 + [ "$cursor" -eq 0 ] && break +done diff --git a/build/run_ci_server.sh b/script/run_ci_server.sh similarity index 100% rename from build/run_ci_server.sh rename to script/run_ci_server.sh diff --git a/build/run_local_server.sh b/script/run_local_server.sh similarity index 98% rename from build/run_local_server.sh rename to script/run_local_server.sh index 324aec35..10a87c5e 100755 --- a/build/run_local_server.sh +++ b/script/run_local_server.sh @@ -40,7 +40,7 @@ done # Generate protobuf files for collab-rt-entity crate. # To run sqlx prepare, we need to build the collab-rt-entity crate first -./build/code_gen.sh +./script/code_gen.sh # Require if there are any changes to the database schema # To build AppFlowy-Cloud binary, we requires the .sqlx files