feat: support mongo connector

coco.yml

@@ -319,6 +319,11 @@ connector:
     interval: 10s
     queue:
       name: indexing_documents
+  mongodb:  
+    enabled: true  
+    interval: 30s  
+    queue:  
+      name: indexing_documents 
   notion:
     enabled: true
     interval: 10s

config/setup/en-US/connector.tpl

@@ -231,6 +231,82 @@ POST $[[SETUP_INDEX_PREFIX]]connector$[[SETUP_SCHEMA_VER]]/$[[SETUP_DOC_TYPE]]/n
   },
   "builtin": true
 }
+POST $[[SETUP_INDEX_PREFIX]]connector$[[SETUP_SCHEMA_VER]]/$[[SETUP_DOC_TYPE]]/mongodb
+{
+  "id" : "mongodb",
+  "created" : "2025-01-12T00:00:00.000000+08:00",
+  "updated" : "2025-01-12T00:00:00.000000+08:00",
+  "name" : "MongoDB Connector",
+  "description" : "Powerful MongoDB database connector supporting incremental/full sync, field mapping (collection-level + global-level), pagination, cluster type optimization, authentication database configuration, projection pushdown, index hints, and other advanced features. Supports standalone, replica set, and sharded cluster deployments.",
+  "category" : "database",
+  "icon" : "/assets/icons/connector/mongodb/icon.png",
+  "tags" : [
+    "nosql",
+    "storage",
+    "database",
+    "document",
+    "mongodb",
+    "incremental_sync",
+    "field_mapping",
+    "pagination",
+    "cluster_optimization",
+    "authentication",
+    "performance"
+  ],
+  "url" : "http://coco.rs/connectors/mongodb",
+  "assets" : {
+    "icons" : {
+      "default" : "/assets/icons/connector/mongodb/icon.png",
+      "collection" : "/assets/icons/connector/mongodb/collection.png",
+      "document" : "/assets/icons/connector/mongodb/document.png",
+      "replica_set" : "/assets/icons/connector/mongodb/replica_set.png",
+      "sharded" : "/assets/icons/connector/mongodb/sharded.png"
+    }
+  },
+  "config": {
+    "connection_uri": "mongodb://username:password@localhost:27017/database",
+    "database": "database_name",
+    "auth_database": "admin",
+    "cluster_type": "standalone",
+    "collections": [
+      {
+        "name": "collection_name",
+        "filter": {"status": "active"},
+        "title_field": "title",
+        "content_field": "content",
+        "category_field": "category",
+        "tags_field": "tags",
+        "url_field": "url",
+        "timestamp_field": "updated_at"
+      }
+    ],
+    "pagination": true,
+    "page_size": 500,
+    "sync_strategy": "incremental",
+    "last_modified_field": "updated_at",
+    "field_mapping": {
+      "enabled": true,
+      "mapping": {
+        "id": "custom_id",
+        "title": "custom_title",
+        "content": "custom_content",
+        "category": "custom_category",
+        "tags": "custom_tags",
+        "url": "custom_url",
+        "metadata": "extra_fields"
+      }
+    },
+    "performance": {
+      "batch_size": 1000,
+      "max_pool_size": 10,
+      "timeout": "30s",
+      "enable_projection": true,
+      "enable_index_hint": true
+    }
+  },
+  "builtin": true
+}
+
 POST $[[SETUP_INDEX_PREFIX]]connector$[[SETUP_SCHEMA_VER]]/$[[SETUP_DOC_TYPE]]/postgresql
 {
   "id" : "postgresql",

config/setup/zh-CN/connector.tpl

@@ -231,6 +231,81 @@ POST $[[SETUP_INDEX_PREFIX]]connector$[[SETUP_SCHEMA_VER]]/$[[SETUP_DOC_TYPE]]/n
   },
   "builtin": true
 }
+POST $[[SETUP_INDEX_PREFIX]]connector$[[SETUP_SCHEMA_VER]]/$[[SETUP_DOC_TYPE]]/mongodb
+{
+  "id" : "mongodb",
+  "created" : "2025-01-12T00:00:00.000000+08:00",
+  "updated" : "2025-01-12T00:00:00.000000+08:00",
+  "name" : "MongoDB 连接器",
+  "description" : "强大的MongoDB数据库连接器，支持增量/全量同步、字段映射（集合级别+全局级别）、分页处理、集群类型优化、认证数据库配置、投影下推、索引提示等高级功能。支持单机、复制集、分片集群部署。",
+  "category" : "database",
+  "icon" : "/assets/icons/connector/mongodb/icon.png",
+  "tags" : [
+    "nosql",
+    "storage",
+    "database",
+    "document",
+    "mongodb",
+    "incremental_sync",
+    "field_mapping",
+    "pagination",
+    "cluster_optimization",
+    "authentication",
+    "performance"
+  ],
+  "url" : "http://coco.rs/connectors/mongodb",
+  "assets" : {
+    "icons" : {
+      "default" : "/assets/icons/connector/mongodb/icon.png",
+      "collection" : "/assets/icons/connector/mongodb/collection.png",
+      "document" : "/assets/icons/connector/mongodb/document.png",
+      "replica_set" : "/assets/icons/connector/mongodb/replica_set.png",
+      "sharded" : "/assets/icons/connector/mongodb/sharded.png"
+    }
+  },
+  "config": {
+    "connection_uri": "mongodb://username:password@localhost:27017/database",
+    "database": "database_name",
+    "auth_database": "admin",
+    "cluster_type": "standalone",
+    "collections": [
+      {
+        "name": "collection_name",
+        "filter": {"status": "active"},
+        "title_field": "title",
+        "content_field": "content",
+        "category_field": "category",
+        "tags_field": "tags",
+        "url_field": "url",
+        "timestamp_field": "updated_at"
+      }
+    ],
+    "pagination": true,
+    "page_size": 500,
+    "sync_strategy": "incremental",
+    "last_modified_field": "updated_at",
+    "field_mapping": {
+      "enabled": true,
+      "mapping": {
+        "id": "custom_id",
+        "title": "custom_title",
+        "content": "custom_content",
+        "category": "custom_category",
+        "tags": "custom_tags",
+        "url": "custom_url",
+        "metadata": "extra_fields"
+      }
+    },
+    "performance": {
+      "batch_size": 1000,
+      "max_pool_size": 10,
+      "timeout": "30s",
+      "enable_projection": true,
+      "enable_index_hint": true
+    }
+  },
+  "builtin": true
+} 
 POST $[[SETUP_INDEX_PREFIX]]connector$[[SETUP_SCHEMA_VER]]/$[[SETUP_DOC_TYPE]]/postgresql
 {
   "id" : "postgresql",

docker/init-mongo.js

@@ -0,0 +1,41 @@
+// MongoDB initialization script for testing  
+db = db.getSiblingDB('coco_test');  
+
+// Create test user  
+db.createUser({  
+  user: 'coco_test',  
+  pwd: 'test_password',  
+  roles: [  
+    {  
+      role: 'readWrite',  
+      db: 'coco_test'  
+    }  
+  ]  
+});  
+
+// Create test collections with sample data  
+db.articles.insertMany([  
+  {  
+    title: "Sample Article 1",  
+    content: "This is sample content for testing",  
+    category: "Technology",  
+    tags: ["mongodb", "database"],  
+    url: "https://example.com/article1",  
+    updated_at: new Date(),  
+    status: "published"  
+  },  
+  {  
+    title: "Sample Article 2",   
+    content: "Another sample content for testing",  
+    category: "Programming",  
+    tags: ["go", "backend"],  
+    url: "https://example.com/article2",  
+    updated_at: new Date(),  
+    status: "draft"  
+  }  
+]);  
+
+// Create indexes for better performance  
+db.articles.createIndex({ "updated_at": 1 });  
+db.articles.createIndex({ "status": 1 });  
+db.articles.createIndex({ "category": 1 });

docker/mongodb-test.yml

@@ -0,0 +1,36 @@
+version: '3.8'  
+
+services:  
+  mongodb:  
+    image: mongo:7.0  
+    container_name: coco-mongodb-test  
+    ports:  
+      - "27017:27017"  
+    environment:  
+      MONGO_INITDB_ROOT_USERNAME: admin  
+      MONGO_INITDB_ROOT_PASSWORD: password  
+      MONGO_INITDB_DATABASE: coco_test  
+    volumes:  
+      - mongodb_data:/data/db  
+      - ./init-mongo.js:/docker-entrypoint-initdb.d/init-mongo.js:ro  
+    networks:  
+      - coco-test  
+
+  mongodb-replica:  
+    image: mongo:7.0  
+    container_name: coco-mongodb-replica-test  
+    ports:  
+      - "27018:27017"  
+    command: mongod --replSet rs0 --bind_ip_all  
+    volumes:  
+      - mongodb_replica_data:/data/db  
+    networks:  
+      - coco-test  
+
+volumes:  
+  mongodb_data:  
+  mongodb_replica_data:  
+
+networks:  
+  coco-test:  
+    driver: bridge

examples/mongodb.yml

@@ -0,0 +1,22 @@
+# MongoDB Connector Default Configuration  
+mongodb:  
+  # Default connection settings  
+  default_timeout: "30s"  
+  default_batch_size: 1000  
+  default_max_pool_size: 10  
+
+  # Default sync settings  
+  default_sync_strategy: "full"  
+
+  # Performance tuning  
+  max_concurrent_collections: 5  
+  memory_gc_interval: 10000  
+
+  # Retry settings  
+  connection_retry_attempts: 3  
+  connection_retry_delay: "30s"  
+
+  # Logging  
+  log_level: "info"  
+  log_slow_queries: true  
+  slow_query_threshold: "5s"

go.mod

@@ -91,6 +91,7 @@ require (
 	github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/flatbuffers v25.2.10+incompatible // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/go-github v17.0.0+incompatible // indirect
@@ -134,6 +135,7 @@ require (
 	github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/mschoch/smat v0.2.0 // indirect
 	github.com/nikolalohinski/gonja v1.5.3 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
@@ -165,15 +167,20 @@ require (
 	github.com/twmb/franz-go/pkg/kmsg v1.11.2 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/yargevad/filepathx v1.0.0 // indirect
 	github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zeebo/blake3 v0.2.4 // indirect
 	gitlab.com/golang-commonmark/html v0.0.0-20191124015941-a22733972181 // indirect
 	gitlab.com/golang-commonmark/linkify v0.0.0-20191026162114-a0c2df6c8f82 // indirect
 	gitlab.com/golang-commonmark/markdown v0.0.0-20211110145824-bf3e522c626a // indirect
 	gitlab.com/golang-commonmark/mdurl v0.0.0-20191124015652-932350d1cb84 // indirect
 	gitlab.com/golang-commonmark/puny v0.0.0-20191124015043-9f83538fa04f // indirect
+	go.mongodb.org/mongo-driver v1.17.4 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
 	go.opentelemetry.io/otel v1.35.0 // indirect

go.sum

@@ -175,6 +175,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q=
 github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/generative-ai-go v0.15.1 h1:n8aQUpvhPOlGVuM2DRkJ2jvx04zpp42B778AROJa+pQ=
@@ -308,6 +310,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
+github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
 github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
 github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
 github.com/nikolalohinski/gonja v1.5.3 h1:GsA+EEaZDZPGJ8JtpeGN78jidhOlxeJROpqMT9fTj9c=
@@ -417,12 +421,20 @@ github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaU
 github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
 github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg=
 github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
 github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=
 github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA=
 github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
 github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4=
+github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
+github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
@@ -444,6 +456,8 @@ gitlab.com/golang-commonmark/puny v0.0.0-20191124015043-9f83538fa04f h1:Wku8eEde
 gitlab.com/golang-commonmark/puny v0.0.0-20191124015043-9f83538fa04f/go.mod h1:Tiuhl+njh/JIg0uS/sOJVYi0x2HEa5rc1OAaVsb5tAs=
 gitlab.com/opennota/wd v0.0.0-20180912061657-c5d65f63c638 h1:uPZaMiz6Sz0PZs3IZJWpU5qHKGNy///1pacZC9txiUI=
 gitlab.com/opennota/wd v0.0.0-20180912061657-c5d65f63c638/go.mod h1:EGRJaqe2eO9XGmFtQCvV3Lm9NLico3UhFwUpCG/+mVU=
+go.mongodb.org/mongo-driver v1.17.4 h1:jUorfmVzljjr0FLzYQsGP8cgN/qzzxlY9Vh0C9KFXVw=
+go.mongodb.org/mongo-driver v1.17.4/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
 go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
 go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw=