From b8d8007f26f1b91ae7855ea74019eb4b66bdec59 Mon Sep 17 00:00:00 2001 From: Khor Shu Heng <32997938+khorshuheng@users.noreply.github.com> Date: Wed, 5 Feb 2025 09:55:56 +0800 Subject: [PATCH] fix: issue:991 unable to import from Notion when using minio (#1203) --- deploy.env | 6 ++++- docker-compose.yml | 1 + libs/database/src/file/s3_client_impl.rs | 25 ++++++++++++++++--- nginx/nginx.conf | 24 ++++++++++++++++++ .../appflowy-collaborate/src/application.rs | 2 ++ services/appflowy-collaborate/src/config.rs | 2 ++ src/application.rs | 2 ++ src/config/config.rs | 2 ++ tests/file_test/mod.rs | 3 +++ 9 files changed, 63 insertions(+), 4 deletions(-) diff --git a/deploy.env b/deploy.env index 1854ccfe..4e056a5f 100644 --- a/deploy.env +++ b/deploy.env @@ -123,7 +123,11 @@ APPFLOWY_S3_MINIO_URL=http://${MINIO_HOST}:${MINIO_PORT} # change this if you ar APPFLOWY_S3_ACCESS_KEY=${AWS_ACCESS_KEY} APPFLOWY_S3_SECRET_KEY=${AWS_SECRET} APPFLOWY_S3_BUCKET=appflowy -#APPFLOWY_S3_REGION=us-east-1 +# Uncomment this if you are using AWS S3 +# APPFLOWY_S3_REGION=us-east-1 +# Uncomment this if you are using the Minio service hosted within this docker compose file +# This is so that, the presigned URL generated by AppFlowy Cloud will use the publicly availabe minio endpoint. +# APPFLOWY_S3_PRESIGNED_URL_ENDPOINT=${FQDN}/minio-api # AppFlowy Cloud Mailer # Note that smtps (TLS) is always required, even for ports other than 465 diff --git a/docker-compose.yml b/docker-compose.yml index e87a16cc..1e59783e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -125,6 +125,7 @@ services: - APPFLOWY_S3_SECRET_KEY=${APPFLOWY_S3_SECRET_KEY} - APPFLOWY_S3_BUCKET=${APPFLOWY_S3_BUCKET} - APPFLOWY_S3_REGION=${APPFLOWY_S3_REGION} + - APPFLOWY_S3_PRESIGNED_URL_ENDPOINT=${APPFLOWY_S3_PRESIGNED_URL_ENDPOINT} - APPFLOWY_MAILER_SMTP_HOST=${APPFLOWY_MAILER_SMTP_HOST} - APPFLOWY_MAILER_SMTP_PORT=${APPFLOWY_MAILER_SMTP_PORT} - APPFLOWY_MAILER_SMTP_USERNAME=${APPFLOWY_MAILER_SMTP_USERNAME} diff --git a/libs/database/src/file/s3_client_impl.rs b/libs/database/src/file/s3_client_impl.rs index 878195ff..aa111e6c 100644 --- a/libs/database/src/file/s3_client_impl.rs +++ b/libs/database/src/file/s3_client_impl.rs @@ -34,12 +34,24 @@ impl S3BucketStorage { pub struct AwsS3BucketClientImpl { client: Client, bucket: String, + endpoint: String, + presigned_url_endpoint: Option, } impl AwsS3BucketClientImpl { - pub fn new(client: Client, bucket: String) -> Self { + pub fn new( + client: Client, + bucket: String, + endpoint: String, + presigned_url_endpoint: Option, + ) -> Self { debug_assert!(!bucket.is_empty()); - AwsS3BucketClientImpl { client, bucket } + AwsS3BucketClientImpl { + client, + bucket, + endpoint, + presigned_url_endpoint, + } } pub async fn gen_presigned_url( @@ -71,7 +83,14 @@ impl AwsS3BucketClientImpl { .await .map_err(|err| AppError::Internal(anyhow!("Generate presigned url failed: {:?}", err)))?; let url = put_object_req.uri().to_string(); - Ok(url) + + let public_url = self + .presigned_url_endpoint + .as_ref() + .map_or(url.clone(), |presigned| { + url.replace(&self.endpoint, presigned) + }); + Ok(public_url) } async fn complete_upload_and_get_metadata( diff --git a/nginx/nginx.conf b/nginx/nginx.conf index c2cee517..e3079a10 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -46,6 +46,9 @@ http { set $appflowy_web_backend "http://appflowy_web:80"; set $appflowy_ai_backend "http://ai:5001"; set $minio_backend "http://minio:9001"; + set $minio_api_backend "http://minio:9000"; + # Host name for minio, used internally within docker compose + set $minio_internal_host "minio:9000"; set $portainer_backend "http://portainer:9000"; set $pgadmin_backend "http://pgadmin:80"; @@ -172,6 +175,7 @@ http { # Minio Web UI # Derive from: https://min.io/docs/minio/linux/integrations/setup-nginx-proxy-with-minio.html # Optional Module, comment this section if you are did not deploy minio in docker-compose.yml + # This endpoint is meant to be used for the MinIO Web UI, accessible via the admin portal location /minio/ { proxy_pass $minio_backend; @@ -198,6 +202,26 @@ http { chunked_transfer_encoding off; } + # Optional Module, comment this section if you are did not deploy minio in docker-compose.yml + # This is used for presigned url, which is needs to be exposed to the AppFlowy client application. + location /minio-api/ { + proxy_pass $minio_api_backend; + + # Set the host to internal host because the presigned url was signed against the internal host + proxy_set_header Host $minio_internal_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + rewrite ^/minio-api/(.*) /$1 break; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + } + # PgAdmin # Optional Module, comment this section if you are did not deploy pgadmin in docker-compose.yml location /pgadmin/ { diff --git a/services/appflowy-collaborate/src/application.rs b/services/appflowy-collaborate/src/application.rs index eb4cfb8a..51311579 100644 --- a/services/appflowy-collaborate/src/application.rs +++ b/services/appflowy-collaborate/src/application.rs @@ -128,6 +128,8 @@ pub async fn init_state(config: &Config, rt_cmd_tx: CLCommandSender) -> Result, pub bucket: String, pub region: String, + pub presigned_url_endpoint: Option, } #[derive(Clone, Debug)] @@ -190,6 +191,7 @@ pub fn get_configuration() -> Result { secret_key: get_env_var("APPFLOWY_S3_SECRET_KEY", "minioadmin").into(), bucket: get_env_var("APPFLOWY_S3_BUCKET", "appflowy"), region: get_env_var("APPFLOWY_S3_REGION", ""), + presigned_url_endpoint: None, }, gotrue: GoTrueSetting { jwt_secret: get_env_var("APPFLOWY_GOTRUE_JWT_SECRET", "hello456").into(), diff --git a/src/application.rs b/src/application.rs index bd35de72..c7f8cf03 100644 --- a/src/application.rs +++ b/src/application.rs @@ -191,6 +191,8 @@ pub async fn init_state(config: &Config, rt_cmd_tx: CLCommandSender) -> Result, pub bucket: String, pub region: String, + pub presigned_url_endpoint: Option, } #[derive(serde::Deserialize, Clone, Debug)] @@ -228,6 +229,7 @@ pub fn get_configuration() -> Result { secret_key: get_env_var("APPFLOWY_S3_SECRET_KEY", "minioadmin").into(), bucket: get_env_var("APPFLOWY_S3_BUCKET", "appflowy"), region: get_env_var("APPFLOWY_S3_REGION", ""), + presigned_url_endpoint: get_env_var_opt("APPFLOWY_S3_PRESIGNED_URL_ENDPOINT"), }, appflowy_ai: AppFlowyAISetting { port: get_env_var("AI_SERVER_PORT", "5001").into(), diff --git a/tests/file_test/mod.rs b/tests/file_test/mod.rs index f06746e8..62b0e222 100644 --- a/tests/file_test/mod.rs +++ b/tests/file_test/mod.rs @@ -44,10 +44,13 @@ impl TestBucket { secret_key: Secret::new(LOCALHOST_MINIO_SECRET_KEY.to_string()), bucket: LOCALHOST_MINIO_BUCKET_NAME.to_string(), region: "".to_string(), + presigned_url_endpoint: None, }; let client = AwsS3BucketClientImpl::new( get_aws_s3_client(&setting).await.unwrap(), setting.bucket.clone(), + LOCALHOST_MINIO_URL.to_string(), + setting.presigned_url_endpoint.clone(), ); Self(client) }