fix: issue:991 unable to import from Notion when using minio (#1203)

deploy.env

@@ -123,7 +123,11 @@ APPFLOWY_S3_MINIO_URL=http://${MINIO_HOST}:${MINIO_PORT} # change this if you ar
 APPFLOWY_S3_ACCESS_KEY=${AWS_ACCESS_KEY}
 APPFLOWY_S3_SECRET_KEY=${AWS_SECRET}
 APPFLOWY_S3_BUCKET=appflowy
-#APPFLOWY_S3_REGION=us-east-1
+# Uncomment this if you are using AWS S3
+# APPFLOWY_S3_REGION=us-east-1
+# Uncomment this if you are using the Minio service hosted within this docker compose file
+# This is so that, the presigned URL generated by AppFlowy Cloud will use the publicly availabe minio endpoint.
+# APPFLOWY_S3_PRESIGNED_URL_ENDPOINT=${FQDN}/minio-api
 
 # AppFlowy Cloud Mailer
 # Note that smtps (TLS) is always required, even for ports other than 465

docker-compose.yml

@@ -125,6 +125,7 @@ services:
       - APPFLOWY_S3_SECRET_KEY=${APPFLOWY_S3_SECRET_KEY}
       - APPFLOWY_S3_BUCKET=${APPFLOWY_S3_BUCKET}
       - APPFLOWY_S3_REGION=${APPFLOWY_S3_REGION}
+      - APPFLOWY_S3_PRESIGNED_URL_ENDPOINT=${APPFLOWY_S3_PRESIGNED_URL_ENDPOINT}
       - APPFLOWY_MAILER_SMTP_HOST=${APPFLOWY_MAILER_SMTP_HOST}
       - APPFLOWY_MAILER_SMTP_PORT=${APPFLOWY_MAILER_SMTP_PORT}
       - APPFLOWY_MAILER_SMTP_USERNAME=${APPFLOWY_MAILER_SMTP_USERNAME}

libs/database/src/file/s3_client_impl.rs

@@ -34,12 +34,24 @@ impl S3BucketStorage {
 pub struct AwsS3BucketClientImpl {
   client: Client,
   bucket: String,
+  endpoint: String,
+  presigned_url_endpoint: Option<String>,
 }
 
 impl AwsS3BucketClientImpl {
-  pub fn new(client: Client, bucket: String) -> Self {
+  pub fn new(
+    client: Client,
+    bucket: String,
+    endpoint: String,
+    presigned_url_endpoint: Option<String>,
+  ) -> Self {
     debug_assert!(!bucket.is_empty());
-    AwsS3BucketClientImpl { client, bucket }
+    AwsS3BucketClientImpl {
+      client,
+      bucket,
+      endpoint,
+      presigned_url_endpoint,
+    }
   }
 
   pub async fn gen_presigned_url(
@@ -71,7 +83,14 @@ impl AwsS3BucketClientImpl {
       .await
       .map_err(|err| AppError::Internal(anyhow!("Generate presigned url failed: {:?}", err)))?;
     let url = put_object_req.uri().to_string();
-    Ok(url)
+
+    let public_url = self
+      .presigned_url_endpoint
+      .as_ref()
+      .map_or(url.clone(), |presigned| {
+        url.replace(&self.endpoint, presigned)
+      });
+    Ok(public_url)
   }
 
   async fn complete_upload_and_get_metadata(

nginx/nginx.conf

@@ -46,6 +46,9 @@ http {
         set $appflowy_web_backend "http://appflowy_web:80";
         set $appflowy_ai_backend "http://ai:5001";
         set $minio_backend "http://minio:9001";
+        set $minio_api_backend "http://minio:9000";
+        # Host name for minio, used internally within docker compose
+        set $minio_internal_host "minio:9000";
         set $portainer_backend "http://portainer:9000";
         set $pgadmin_backend "http://pgadmin:80";
 
@@ -172,6 +175,7 @@ http {
         # Minio Web UI
         # Derive from: https://min.io/docs/minio/linux/integrations/setup-nginx-proxy-with-minio.html
         # Optional Module, comment this section if you are did not deploy minio in docker-compose.yml
+        # This endpoint is meant to be used for the MinIO Web UI, accessible via the admin portal
         location /minio/ {
             proxy_pass $minio_backend;
 
@@ -198,6 +202,26 @@ http {
             chunked_transfer_encoding off;
         }
 
+        # Optional Module, comment this section if you are did not deploy minio in docker-compose.yml
+        # This is used for presigned url, which is needs to be exposed to the AppFlowy client application.
+        location /minio-api/ {
+            proxy_pass $minio_api_backend;
+
+            # Set the host to internal host because the presigned url was signed against the internal host
+            proxy_set_header Host $minio_internal_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            rewrite ^/minio-api/(.*) /$1 break;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+        }
+
         # PgAdmin
         # Optional Module, comment this section if you are did not deploy pgadmin in docker-compose.yml
         location /pgadmin/ {

services/appflowy-collaborate/src/application.rs

@@ -128,6 +128,8 @@ pub async fn init_state(config: &Config, rt_cmd_tx: CLCommandSender) -> Result<A
   let s3_client = AwsS3BucketClientImpl::new(
     get_aws_s3_client(&config.s3).await?,
     config.s3.bucket.clone(),
+    config.s3.minio_url.clone(),
+    config.s3.presigned_url_endpoint.clone(),
   );
 
   let collab_access_control = CollabAccessControlImpl::new(access_control.clone());

services/appflowy-collaborate/src/config.rs

@@ -30,6 +30,7 @@ pub struct S3Setting {
   pub secret_key: Secret<String>,
   pub bucket: String,
   pub region: String,
+  pub presigned_url_endpoint: Option<String>,
 }
 
 #[derive(Clone, Debug)]
@@ -190,6 +191,7 @@ pub fn get_configuration() -> Result<Config, anyhow::Error> {
       secret_key: get_env_var("APPFLOWY_S3_SECRET_KEY", "minioadmin").into(),
       bucket: get_env_var("APPFLOWY_S3_BUCKET", "appflowy"),
       region: get_env_var("APPFLOWY_S3_REGION", ""),
+      presigned_url_endpoint: None,
     },
     gotrue: GoTrueSetting {
       jwt_secret: get_env_var("APPFLOWY_GOTRUE_JWT_SECRET", "hello456").into(),

src/application.rs

@@ -191,6 +191,8 @@ pub async fn init_state(config: &Config, rt_cmd_tx: CLCommandSender) -> Result<A
   let s3_client = AwsS3BucketClientImpl::new(
     get_aws_s3_client(&config.s3).await?,
     config.s3.bucket.clone(),
+    config.s3.minio_url.clone(),
+    config.s3.presigned_url_endpoint.clone(),
   );
   let bucket_storage = Arc::new(S3BucketStorage::from_bucket_impl(
     s3_client.clone(),

src/config/config.rs

@@ -59,6 +59,7 @@ pub struct S3Setting {
   pub secret_key: Secret<String>,
   pub bucket: String,
   pub region: String,
+  pub presigned_url_endpoint: Option<String>,
 }
 
 #[derive(serde::Deserialize, Clone, Debug)]
@@ -228,6 +229,7 @@ pub fn get_configuration() -> Result<Config, anyhow::Error> {
       secret_key: get_env_var("APPFLOWY_S3_SECRET_KEY", "minioadmin").into(),
       bucket: get_env_var("APPFLOWY_S3_BUCKET", "appflowy"),
       region: get_env_var("APPFLOWY_S3_REGION", ""),
+      presigned_url_endpoint: get_env_var_opt("APPFLOWY_S3_PRESIGNED_URL_ENDPOINT"),
     },
     appflowy_ai: AppFlowyAISetting {
       port: get_env_var("AI_SERVER_PORT", "5001").into(),

tests/file_test/mod.rs

@@ -44,10 +44,13 @@ impl TestBucket {
       secret_key: Secret::new(LOCALHOST_MINIO_SECRET_KEY.to_string()),
       bucket: LOCALHOST_MINIO_BUCKET_NAME.to_string(),
       region: "".to_string(),
+      presigned_url_endpoint: None,
     };
     let client = AwsS3BucketClientImpl::new(
       get_aws_s3_client(&setting).await.unwrap(),
       setting.bucket.clone(),
+      LOCALHOST_MINIO_URL.to_string(),
+      setting.presigned_url_endpoint.clone(),
     );
     Self(client)
   }