diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..aeae7e116 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,77 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +--- + +## 🚀 Development Commands + +### 🏗️ Development Workflow +The project is structured with a distinct frontend (Vue/Vite) and a backend (FastAPI/Python) component. + +**1. Frontend/Client (Vue/Vite)** +The client-side application manages the UI and handles interactions with the backend API. All client-side commands are defined in `frontend/package.json`. + +* **Run/Develop:** `npm run dev` + * *Purpose:* Runs the development server (`vue-tsc -b && vite`) for hot reloading and development testing. +* **Build Production Assets:** `npm run build` + * *Purpose:* Compiles the client for production (`vue-tsc -b &&NODE_OPTIONS=--max_old_space_size=4096 vite build`). +* **Linting:** `npm run lint` + * *Purpose:* Runs ESLint checks across all Vue, JavaScript, and TypeScript files. + +**2. Backend/API (Python/FastAPI)** +The core API logic is located in the `backend` directory. Database migrations must be handled via Alembic. + +* **Database Migrations:** The `backend/alembic` directory contains the history of all schema changes. To apply or check migrations, use the appropriate `alembic` command (e.g., `alembic upgrade head`). +* **Running Tests (Hypothetical):** While a dedicated test runner command was not found, standard practice suggests running tests using a command like `pytest` within the `backend` context. + +### 🐳 Deployment +The system is containerized and deployed using Docker. +* **Docker Command:** + ```bash + docker run -d \ + --name sqlbot \ + --restart unless-stopped \ + -p 8000:8000 \ + -p 8001:8001 \ + -v ./data/sqlbot/excel:/opt/sqlbot/data/excel \ + -v ./data/sqlbot/file:/opt/sqlbot/data/file \ + -v ./data/sqlbot/images:/opt/sqlbot/images \ + -v ./data/sqlbot/logs:/opt/sqlbot/app/logs \ + -v ./data/postgresql:/var/lib/postgresql/data \ + --privileged=true \ + dataease/sqlbot + ``` + +### 🗄️ MongoDB (Queryable Datasource) +MongoDB is supported as a **queryable datasource** type in the datasource system (alongside PostgreSQL, MySQL, Elasticsearch, etc.). Users create a MongoDB datasource and ChatBI answers questions by generating and executing MongoDB aggregation pipelines against it. + +* **How it works:** The LLM generates a MongoDB aggregation pipeline (a JSON array) which is stored in the record's `sql` field and executed by `backend/apps/db/mongo_engine.py` via pymongo. This mirrors how the Elasticsearch (`es`) datasource works with ES DSL. +* **Type registration:** `backend/apps/db/constant.py` (`DB.mongo`). Connection/execution dispatch lives in `backend/apps/db/db.py`; driver helpers in `backend/apps/db/mongo_engine.py`. +* **LLM prompt:** `backend/templates/sql_examples/MongoDB.yaml` teaches the model to output MongoDB aggregation pipelines instead of SQL. +* **Dependency:** `pymongo` is declared in `backend/pyproject.toml` and installed by `uv sync`. +* **Frontend:** MongoDB appears in `frontend/src/views/ds/js/ds-type.ts` with its own config form. +* **Not a runtime store:** SQLBot's own runtime data stays in PostgreSQL only. MongoDB is only ever a datasource a user connects to; it is not used for SQLBot's internal persistence. + +--- + +## 📐 Architecture Overview + +**Core Concept:** +SQLBot is an intelligent Question-Answering and ChatBI (Chat Business Intelligence) system. It uses Large Language Models (LLMs) combined with Retrieval-Augmented Generation (RAG) to enable users to perform data analysis using natural language, translating prompts into accurate SQL queries. + +**Structure:** +1. **`frontend/`:** This directory contains the client-side Single Page Application (SPA). It is built with Vue/Vite and handles the user interface, displaying results and accepting natural language input. +2. **`backend/`:** This directory houses the primary API logic, built using FastAPI. It is responsible for: + * Receiving user prompts. + * Managing RAG processes and database interactions. + * Orchestrating the LLM calls and Text-to-SQL transformation. +3. **`backend/alembic/`:** This directory manages the PostgreSQL database schema migrations. All historical schema changes are tracked here, pointing to the database structure. + +**Data Flow:** +User input -> `frontend/` -> API endpoint (via `backend/`) -> Orchestration/RAG -> Database Querying/SQL Generation -> Result -> `backend/` -> `frontend/` (Display). + +**Key System Components:** +* **LLM Integration:** Supports multiple providers (OpenAI, Gemini, etc.) with an OpenAI compatibility layer. +* **Permissions/Security:** Features work-space level resource isolation and fine-grained data permission configuration. +* **Deployment:** The entire stack is containerized via Docker. \ No newline at end of file diff --git a/backend/apps/chat/curd/chat.py b/backend/apps/chat/curd/chat.py index 292a3d5cf..412a9246f 100644 --- a/backend/apps/chat/curd/chat.py +++ b/backend/apps/chat/curd/chat.py @@ -548,7 +548,11 @@ def format_record(record: ChatRecordResult): pass if record.sql and record.sql.strip() != '': try: - _dict['sql'] = sqlparse.format(record.sql, reindent=True) + if record.sql.strip()[0] == '{': + # MongoDB 聚合管道 JSON，直接美化输出 + _dict['sql'] = orjson.dumps(orjson.loads(record.sql), option=orjson.OPT_INDENT_2).decode() + else: + _dict['sql'] = sqlparse.format(record.sql, reindent=True) except Exception: pass diff --git a/backend/apps/chat/models/chat_model.py b/backend/apps/chat/models/chat_model.py index f169cfb17..42e2a8aa2 100644 --- a/backend/apps/chat/models/chat_model.py +++ b/backend/apps/chat/models/chat_model.py @@ -270,16 +270,21 @@ def sql_sys_question(self, db_type: Union[str, DB], enable_query_limit: bool = T _example_answer_3 = _sql_template['example_answer_3_with_limit'] if enable_query_limit else _sql_template[ 'example_answer_3'] - templates['system'] = _base_template['system'].format(lang=self.lang, process_check=_process_check, - sqlbot_name=self.sqlbot_name) - templates['rules'] = _base_template['generate_rules'].format(lang=self.lang, - sqlbot_name=self.sqlbot_name, - base_sql_rules=_base_sql_rules, - basic_sql_examples=_sql_examples, - example_engine=_example_engine, - example_answer_1=_example_answer_1, - example_answer_2=_example_answer_2, - example_answer_3=_example_answer_3) + # NoSQL 数据源（如 MongoDB）可提供专属的 system / generate_rules 提示词， + # 否则回退到关系型数据库的通用模板 + _sql_system = _sql_template.get('system') or _base_template['system'] + _sql_generate_rules = _sql_template.get('generate_rules') or _base_template['generate_rules'] + + templates['system'] = _sql_system.format(lang=self.lang, process_check=_process_check, + sqlbot_name=self.sqlbot_name) + templates['rules'] = _sql_generate_rules.format(lang=self.lang, + sqlbot_name=self.sqlbot_name, + base_sql_rules=_base_sql_rules, + basic_sql_examples=_sql_examples, + example_engine=_example_engine, + example_answer_1=_example_answer_1, + example_answer_2=_example_answer_2, + example_answer_3=_example_answer_3) templates['schema'] = _base_template['generate_basic_info'].format(engine=self.engine, schema=self.db_schema, sample_data=self.sample_data) @@ -297,14 +302,16 @@ def sql_sys_question(self, db_type: Union[str, DB], enable_query_limit: bool = T return templates - def sql_user_question(self, current_time: str, change_title: bool): + def sql_user_question(self, db_type: Union[str, DB] = None, current_time: str = '', change_title: bool = False): _question = self.question if self.regenerate_record_id: _question = get_sql_template()['regenerate_hint'] + self.question - return get_sql_template()['user'].format(lang=self.lang, engine=self.engine, schema=self.db_schema, - question=_question, - rule=self.rule, current_time=current_time, error_msg=self.error_msg, - change_title=change_title) + # NoSQL 数据源（如 MongoDB）可提供专属的 user 提示词，否则回退到通用模板 + _user_template = get_sql_example_template(db_type).get('user') or get_sql_template()['user'] + return _user_template.format(lang=self.lang, engine=self.engine, schema=self.db_schema, + question=_question, + rule=self.rule, current_time=current_time, error_msg=self.error_msg, + change_title=change_title) def chart_sys_question(self): templates: dict[str, str] = { diff --git a/backend/apps/chat/task/llm.py b/backend/apps/chat/task/llm.py index e9ef80420..f4a0b255c 100644 --- a/backend/apps/chat/task/llm.py +++ b/backend/apps/chat/task/llm.py @@ -71,6 +71,15 @@ def extract_tables_from_sql(sql: str, ds_type: str = None) -> set: """从 SQL 中提取真实表名（排除 CTE 别名）""" tables = set() + # MongoDB 查询是聚合管道 JSON，集合名存放在 "collection" 字段 + if ds_type and ds_type.lower() == 'mongo': + try: + parsed = json.loads(sql) + if isinstance(parsed, dict) and parsed.get('collection'): + tables.add(parsed['collection']) + except Exception: + pass + return tables dialect = get_sqlglot_dialect(ds_type) try: statements = sqlglot.parse(sql, dialect=dialect) @@ -783,7 +792,8 @@ def select_datasource(self, _session: Session): def generate_sql(self, _session: Session): # append current question self.sql_message.append(HumanMessage( - self.chat_question.sql_user_question(current_time=datetime.now().strftime('%Y-%m-%d %H:%M:%S'), + self.chat_question.sql_user_question(db_type=self.ds.type if self.ds else None, + current_time=datetime.now().strftime('%Y-%m-%d %H:%M:%S'), change_title=self.change_title))) self.current_logs[OperationEnum.GENERATE_SQL] = start_log(session=_session, @@ -995,6 +1005,29 @@ def generate_chart(self, _session: Session, chart_type: Optional[str] = '', sche reasoning_content=full_thinking_text, token_usage=token_usage) + @staticmethod + def build_no_sql_message(data: dict) -> str: + """模型未生成 SQL 时，拼装完整的用户可读说明。 + + 兼容两类返回： + 1. 标准约定：{"success": false, "message": "..."} + 2. 模型自创的非标准结构（常见于本地模型）：{"description": "...", "schema_details": [...]} + 将说明文字与字段明细合并展示，避免模型返回的完整信息（字段名/类型/说明）被丢弃。 + """ + message = data.get('message') or data.get('description') or 'Unable to generate SQL' + schema_details = data.get('schema_details') or data.get('fields') + if schema_details: + lines = [message] + for field in schema_details: + if isinstance(field, dict): + name = field.get('field_name') or field.get('name') or field.get('collection_name') + dtype = field.get('data_type') or field.get('type') or '' + desc = field.get('description') or '' + if name: + lines.append(f"- {name}: {dtype} - {desc}".rstrip(' -')) + message = '\n'.join(lines) + return message + def check_sql(self, session: Session, res: str, operate: OperationEnum) -> tuple[str, Optional[list]]: json_str = extract_nested_json(res) @@ -1006,14 +1039,34 @@ def check_sql(self, session: Session, res: str, operate: OperationEnum) -> tuple 'traceback': "SQL answer is not a valid json object:\n" + res}).decode()) sql: str data: dict + is_mongo = self.ds.type.lower() == 'mongo' try: data = orjson.loads(json_str) - if data['success']: - sql = data['sql'] + success = data.get('success') + if success is None and is_mongo: + # 本地模型可能不输出 success 字段，但提供了聚合管道/集合名，视为可执行查询 + success = bool(data.get('sql') or data.get('aggregation_pipeline') or data.get('pipeline')) + + if success: + sql = data.get('sql') + if (not sql or not sql.strip()) and is_mongo: + # 兼容模型将管道拆成 aggregation_pipeline + collection_name/collection 返回的情况， + # 拼装成标准查询 {"collection": ..., "pipeline": [...]} + collection = data.get('collection_name') or data.get('collection') + pipeline = data.get('aggregation_pipeline') or data.get('pipeline') + if collection and isinstance(pipeline, list): + sql = json.dumps({'collection': collection, 'pipeline': pipeline}, ensure_ascii=False) + if is_mongo and not data.get('tables'): + try: + _coll = json.loads(sql).get('collection') if sql and sql.strip() else None + if _coll: + data['tables'] = [_coll] + except Exception: + pass else: - message = data['message'] - raise SingleMessageError(message) + # 模型未生成 SQL：可能是不需要查询（如询问表结构/集合结构），优先展示其完整说明 + raise SingleMessageError(self.build_no_sql_message(data)) except SingleMessageError as e: trigger_log_error(session, log) raise e @@ -1022,7 +1075,7 @@ def check_sql(self, session: Session, res: str, operate: OperationEnum) -> tuple raise SingleMessageError(orjson.dumps({'message': 'Cannot parse sql from answer', 'traceback': "Cannot parse sql from answer:\n" + res}).decode()) - if sql.strip() == '': + if not sql or sql.strip() == '': trigger_log_error(session, log) raise SingleMessageError("SQL query is empty") return sql, data.get('tables') @@ -1345,6 +1398,9 @@ def run_task(self, in_chat: bool = True, stream: bool = True, dynamic_sql_result = self.generate_assistant_dynamic_sql(_session, sql, tables) sqlbot_temp_sql_text = dynamic_sql_result.get( 'sqlbot_temp_sql_text') if dynamic_sql_result else None + elif self.ds.type.lower() == 'mongo': + # 行级权限过滤会把 WHERE 子句拼接到 SQL，对聚合管道 JSON 不适用 + sql_result = None else: sql_result = self.generate_filter(_session, sql, tables) # maybe no sql and tables @@ -1366,7 +1422,13 @@ def run_task(self, in_chat: bool = True, stream: bool = True, if not stream: json_result['sql'] = sql - format_sql = sqlparse.format(sql, reindent=True) + if self.ds.type.lower() == 'mongo': + try: + format_sql = json.dumps(json.loads(sql), indent=2, ensure_ascii=False) + except Exception: + format_sql = sql + else: + format_sql = sqlparse.format(sql, reindent=True) if in_chat: yield 'data:' + orjson.dumps({'content': format_sql, 'type': 'sql'}).decode() + '\n\n' else: @@ -1514,9 +1576,12 @@ def run_task(self, in_chat: bool = True, stream: bool = True, yield json_result except Exception as e: - traceback.print_exc() error_msg: str if isinstance(e, SingleMessageError): + # SingleMessageError 是预期内的优雅失败路径（例如模型无需生成查询时的说明性回答， + # 如询问表结构/集合结构、无法生成查询等）。其 message 已作为回答直接展示给用户并持久化， + # 无需打印整段堆栈，避免把正常回答误报为系统异常。 + SQLBotLogUtil.warning(f"graceful answer message: {e}") error_msg = str(e) elif isinstance(e, SQLBotDBConnectionError): error_msg = orjson.dumps( @@ -1525,6 +1590,7 @@ def run_task(self, in_chat: bool = True, stream: bool = True, error_msg = orjson.dumps( {'message': 'Execute SQL Failed', 'traceback': str(e), 'type': 'exec-sql-err'}).decode() else: + traceback.print_exc() error_msg = orjson.dumps({'message': str(e), 'traceback': traceback.format_exc(limit=1)}).decode() if _session: self.save_error(session=_session, message=error_msg) @@ -1690,11 +1756,13 @@ def run_analysis_or_predict_task(self, action_type: str, in_chat: bool = True, s if not stream: yield json_result except Exception as e: - traceback.print_exc() error_msg: str if isinstance(e, SingleMessageError): + # 预期内的优雅失败路径：message 直接展示给用户，无需打印整段堆栈 + SQLBotLogUtil.warning(f"graceful answer message: {e}") error_msg = str(e) else: + traceback.print_exc() error_msg = orjson.dumps({'message': str(e), 'traceback': traceback.format_exc(limit=1)}).decode() if _session: self.save_error(session=_session, message=error_msg) diff --git a/backend/apps/datasource/crud/datasource.py b/backend/apps/datasource/crud/datasource.py index 11720a783..b17c7d543 100644 --- a/backend/apps/datasource/crud/datasource.py +++ b/backend/apps/datasource/crud/datasource.py @@ -376,6 +376,9 @@ def preview(session: SessionDep, current_user: CurrentUser, id: int, data: Table sql = f"""SELECT "{'", "'.join(fields)}" FROM "{table.table_name}" {where} LIMIT 100""" + elif ds.type == "mongo": + # 行级权限过滤对 MongoDB 聚合管道不适用 + sql = json.dumps({"collection": table.table_name, "pipeline": [{"$limit": 100}]}, ensure_ascii=False) return exec_sql(ds, sql, True) @@ -391,7 +394,12 @@ def fieldEnum(session: SessionDep, id: int): return [] db = DB.get_db(ds.type) - sql = f"""SELECT DISTINCT {db.prefix}{field.field_name}{db.suffix} FROM {db.prefix}{table.table_name}{db.suffix}""" + if ds.type == "mongo": + sql = json.dumps({"collection": table.table_name, + "pipeline": [{"$group": {"_id": f"${field.field_name}"}}, {"$limit": 100}]}, + ensure_ascii=False) + else: + sql = f"""SELECT DISTINCT {db.prefix}{field.field_name}{db.suffix} FROM {db.prefix}{table.table_name}{db.suffix}""" res = exec_sql(ds, sql, True) return [item.get(res.get('fields')[0]) for item in res.get('data')] @@ -469,13 +477,15 @@ def get_table_sample_data(ds: CoreDatasource, table_name: str, fields: list) -> query = f"SELECT {','.join(field_names)} FROM \"{table_name}\" WHERE ROWNUM <= 3" elif equals_ignore_case(ds.type, "dm"): query = f"SELECT {','.join(field_names)} FROM \"{table_name}\" WHERE ROWNUM <= 3" + elif equals_ignore_case(ds.type, "mongo"): + # MongoDB 采样使用聚合管道而非 SQL + query = json.dumps({"collection": table_name, "pipeline": [{"$limit": 3}]}, ensure_ascii=False) else: query = f"SELECT {','.join(field_names)} FROM {prefix}{table_name}{suffix} LIMIT 3" try: result = exec_sql(ds=ds, sql=query, origin_column=True) if result and result.get('data') and len(result['data']) > 0: - import json # Truncate long string values for readability json_rows = [] for row in result['data'][:3]: @@ -511,7 +521,8 @@ def get_tables_sample_data(session: SessionDep, current_user: CurrentUser, ds: C if obj.fields: sample = get_table_sample_data(ds, obj.table.table_name, obj.fields) if sample: - sample_data_parts.append(f"# Table: {obj.table.table_name}\n{sample}") + table_mark = "Collection" if equals_ignore_case(ds.type, "mongo") else "Table" + sample_data_parts.append(f"# {table_mark}: {obj.table.table_name}\n{sample}") return "\n".join(sample_data_parts) @@ -533,7 +544,13 @@ def get_table_schema(session: SessionDep, current_user: CurrentUser, ds: CoreDat schema_table = '' no_schema_types = ["mysql", "es", "sqlite", "hive", "doris", "starrocks"] - schema_table += f"# Table: {db_name}.{obj.table.table_name}" if ds.type not in no_schema_types and db_name else f"# Table: {obj.table.table_name}" + # MongoDB 使用"集合"而非"表"，与 MongoDB.yaml 中的术语保持一致 + is_mongo = equals_ignore_case(ds.type, "mongo") + table_mark = "Collection" if is_mongo else "Table" + if (not is_mongo and ds.type in no_schema_types) or not db_name: + schema_table += f"# {table_mark}: {obj.table.table_name}" + else: + schema_table += f"# {table_mark}: {db_name}.{obj.table.table_name}" table_comment = '' if obj.table.custom_comment: table_comment = obj.table.custom_comment.strip() diff --git a/backend/apps/db/constant.py b/backend/apps/db/constant.py index dcec81901..1693695a0 100644 --- a/backend/apps/db/constant.py +++ b/backend/apps/db/constant.py @@ -23,6 +23,7 @@ class DB(Enum): doris = ('doris', 'Apache Doris', '`', '`', ConnectType.py_driver, 'Doris', []) es = ('es', 'Elasticsearch', '"', '"', ConnectType.py_driver, 'Elasticsearch', []) kingbase = ('kingbase', 'Kingbase', '"', '"', ConnectType.py_driver, 'Kingbase', []) + mongo = ('mongo', 'MongoDB', '"', '"', ConnectType.py_driver, 'MongoDB', []) sqlServer = ('sqlServer', 'Microsoft SQL Server', '[', ']', ConnectType.sqlalchemy, 'Microsoft_SQL_Server', []) mysql = ('mysql', 'MySQL', '`', '`', ConnectType.sqlalchemy, 'MySQL', ['local_infile']) oracle = ('oracle', 'Oracle', '"', '"', ConnectType.sqlalchemy, 'Oracle', []) diff --git a/backend/apps/db/db.py b/backend/apps/db/db.py index 7f19854d1..5117a8f8b 100644 --- a/backend/apps/db/db.py +++ b/backend/apps/db/db.py @@ -31,6 +31,7 @@ from common.utils.utils import SQLBotLogUtil, equals_ignore_case from fastapi import HTTPException from apps.db.es_engine import get_es_connect, get_es_index, get_es_fields, get_es_data_by_http +from apps.db.mongo_engine import get_mongo_collections, get_mongo_fields, get_mongo_data, check_mongo_connection from common.core.config import settings import sqlglot from sqlglot import expressions as exp @@ -386,6 +387,13 @@ def check_connection(trans: Optional[Trans], ds: CoreDatasource | AssistantOutDs else: SQLBotLogUtil.info("failed") return False + elif equals_ignore_case(ds.type, 'mongo'): + if check_mongo_connection(conf): + SQLBotLogUtil.info("success") + return True + else: + SQLBotLogUtil.info("failed") + return False # else: # conn = get_ds_engine(ds) # try: @@ -536,6 +544,10 @@ def get_tables(ds: CoreDatasource): res = get_es_index(conf) res_list = [TableSchema(*item) for item in res] return res_list + elif equals_ignore_case(ds.type, 'mongo'): + res = get_mongo_collections(conf) + res_list = [TableSchema(*item) for item in res] + return res_list elif equals_ignore_case(ds.type, 'hive'): with get_driver_connection(ds) as conn, conn.cursor() as cursor: cursor.execute(sql) @@ -586,6 +598,10 @@ def get_fields(ds: CoreDatasource, table_name: str = None): res = get_es_fields(conf, table_name) res_list = [ColumnSchema(*item) for item in res] return res_list + elif equals_ignore_case(ds.type, 'mongo'): + res = get_mongo_fields(conf, table_name) + res_list = [ColumnSchema(*item) for item in res] + return res_list elif equals_ignore_case(ds.type, 'hive'): with get_driver_pool(ds).connection() as conn, conn.cursor() as cursor: cursor.execute(sql) @@ -881,6 +897,28 @@ def exec_sql(ds: CoreDatasource | AssistantOutDsSchema, sql: str, origin_column= "sql": bytes.decode(base64.b64encode(bytes(sql, 'utf-8')))} except Exception as ex: raise Exception(str(ex)) + elif equals_ignore_case(ds.type, 'mongo'): + try: + res, raw_columns = get_mongo_data(conf, sql) + columns = raw_columns if origin_column else [col.lower() for col in raw_columns] + fields_info = [] + for idx, col in enumerate(columns): + is_numeric = False + for row in res: + val = row[idx] + if val is None: + continue + is_numeric = isinstance(val, (int, float)) and not isinstance(val, bool) + break + fields_info.append({"name": col, "is_numeric": is_numeric}) + result_list = [ + {str(columns[i]): convert_value(value) for i, value in enumerate(tuple_item)} for tuple_item in + res + ] + return {"fields": columns, "data": result_list, "fields_info": fields_info, + "sql": bytes.decode(base64.b64encode(bytes(sql, 'utf-8')))} + except Exception as ex: + raise ParseSQLResultError(str(ex)) elif equals_ignore_case(ds.type, 'hive'): with get_driver_pool(ds).connection() as conn, conn.cursor() as cursor: try: @@ -1066,6 +1104,26 @@ def check_sql_read(sql: str, ds: CoreDatasource | AssistantOutDsSchema) -> tuple 检查 SQL 是否为安全的只读查询 返回: (是否安全, 错误原因) """ + # MongoDB 查询是聚合管道 JSON 而非 SQL，单独校验并拒绝写阶段 + if equals_ignore_case(ds.type, 'mongo'): + try: + parsed = json.loads(sql) + except Exception as e: + raise ValueError(f"MongoDB query is not valid JSON: {e}") + if isinstance(parsed, dict): + pipeline = parsed.get('pipeline', []) + elif isinstance(parsed, list): + pipeline = parsed + else: + raise ValueError("MongoDB query must be a JSON object or array") + if not isinstance(pipeline, list): + raise ValueError("MongoDB 'pipeline' must be an array") + for stage in pipeline: + if isinstance(stage, dict): + for op in stage.keys(): + if op in ('$out', '$merge'): + return False, f"MongoDB aggregation contains write stage: {op}" + return True, "" try: normalized_sql = sql.strip().lstrip("(").strip() first_keyword = normalized_sql.split(None, 1)[0].upper() if normalized_sql else "" diff --git a/backend/apps/db/db_sql.py b/backend/apps/db/db_sql.py index 496075378..a5250b410 100644 --- a/backend/apps/db/db_sql.py +++ b/backend/apps/db/db_sql.py @@ -29,7 +29,7 @@ def get_version_sql(ds: CoreDatasource, conf: DatasourceConf): return """ SELECT * FROM v$version """ - elif equals_ignore_case(ds.type, "redshift", "sqlite", "hive"): + elif equals_ignore_case(ds.type, "redshift", "sqlite", "hive", "mongo"): return '' @@ -162,6 +162,9 @@ def get_table_sql(ds: CoreDatasource, conf: DatasourceConf, db_version: str = '' """, conf.dbSchema elif equals_ignore_case(ds.type, "es"): return "", None + elif equals_ignore_case(ds.type, "mongo"): + # MongoDB has no tables; collections are listed via mongo_engine. + return "", None elif equals_ignore_case(ds.type, "hive"): return """ SHOW TABLES @@ -316,6 +319,9 @@ def get_field_sql(ds: CoreDatasource, conf: DatasourceConf, table_name: str = No return sql1 + sql2, conf.dbSchema, table_name elif equals_ignore_case(ds.type, "es"): return "", None, None + elif equals_ignore_case(ds.type, "mongo"): + # MongoDB has no tables/columns; fields are inferred via mongo_engine. + return "", None, None elif equals_ignore_case(ds.type, "hive"): sql1 = f"DESCRIBE {table_name}" return sql1, None, None diff --git a/backend/apps/db/mongo_engine.py b/backend/apps/db/mongo_engine.py new file mode 100644 index 000000000..13580f4c1 --- /dev/null +++ b/backend/apps/db/mongo_engine.py @@ -0,0 +1,151 @@ +# Author: Junjun +# Date: 2025/9/9 + +import json +import urllib.parse +from datetime import datetime, date +from decimal import Decimal + +from bson import ObjectId +from pymongo import MongoClient + +from apps.datasource.models.datasource import DatasourceConf +from common.error import SingleMessageError + + +def get_mongo_uri(conf: DatasourceConf) -> str: + """Build a MongoDB URI from the datasource config. + + ``extraJdbc`` holds optional URI options (e.g. ``authSource=admin``). + """ + username = urllib.parse.quote(conf.username) if conf.username else '' + password = urllib.parse.quote(conf.password) if conf.password else '' + auth = f"{username}:{password}@" if (username or password) else '' + port = conf.port if conf.port else 27017 + uri = f"mongodb://{auth}{conf.host}:{port}/" + if conf.extraJdbc: + uri += f"?{conf.extraJdbc}" + return uri + + +def get_mongo_client(conf: DatasourceConf) -> MongoClient: + """Return a MongoClient for the datasource. + + MongoClient is lazy; the server-selection timeout bounds the wait when the + host is unreachable. + """ + return MongoClient(get_mongo_uri(conf), + serverSelectionTimeoutMS=(conf.timeout or 30) * 1000, + maxPoolSize=(conf.poolSize or 5)) + + +def check_mongo_connection(conf: DatasourceConf) -> bool: + """Ping the MongoDB server to verify the datasource is reachable.""" + try: + get_mongo_client(conf).admin.command('ping') + return True + except Exception: + return False + + +# get tables (collections) +def get_mongo_collections(conf: DatasourceConf): + """List collection names as (name, comment) tuples; Mongo has no comments.""" + client = get_mongo_client(conf) + db = client[conf.database] + return [(name, '') for name in db.list_collection_names()] + + +def _bson_type_name(value) -> str: + if value is None: + return 'null' + if isinstance(value, bool): + return 'boolean' + if isinstance(value, int): + return 'int' + if isinstance(value, float): + return 'float' + if isinstance(value, Decimal): + return 'decimal' + if isinstance(value, datetime): + return 'datetime' + if isinstance(value, date): + return 'date' + if isinstance(value, ObjectId): + return 'object_id' + if isinstance(value, dict): + return 'object' + if isinstance(value, list): + return 'array' + return 'string' + + +# get fields +def get_mongo_fields(conf: DatasourceConf, table_name: str): + """Infer field names/types from a sampled document of the collection.""" + client = get_mongo_client(conf) + coll = client[conf.database][table_name] + doc = coll.find_one() + if not doc: + return [] + return [(key, _bson_type_name(value), _bson_type_name(value)) for key, value in doc.items()] + + +def _to_serializable(value): + """Recursively convert BSON types (ObjectId, dict, list) to JSON-safe values.""" + if isinstance(value, ObjectId): + return str(value) + if isinstance(value, dict): + return {key: _to_serializable(v) for key, v in value.items()} + if isinstance(value, (list, tuple)): + return [_to_serializable(v) for v in value] + return value + + +# execute aggregation pipeline +def get_mongo_data(conf: DatasourceConf, sql: str): + """Execute a MongoDB aggregation pipeline and return ``(rows, columns)``. + + ``sql`` is a JSON string in the form:: + + {"collection": "", "pipeline": [ {stage}, ... ]} + + ``rows`` is a list of tuples (one per result document, aligned to + ``columns``); ``columns`` is the ordered union of keys across documents. + """ + try: + parsed = json.loads(sql) + except json.JSONDecodeError as e: + raise SingleMessageError(f"MongoDB query is not valid JSON: {e}") + + if isinstance(parsed, dict): + collection_name = parsed.get('collection') + pipeline = parsed.get('pipeline', []) + elif isinstance(parsed, list): + collection_name = None + pipeline = parsed + else: + raise SingleMessageError("MongoDB query must be a JSON object or array") + + if not collection_name: + raise SingleMessageError("MongoDB query must include a 'collection' name") + if not isinstance(pipeline, list): + raise SingleMessageError("MongoDB 'pipeline' must be an array") + + client = get_mongo_client(conf) + coll = client[conf.database][collection_name] + try: + docs = list(coll.aggregate(pipeline)) + except Exception as e: + raise SingleMessageError(f"MongoDB aggregation failed: {e}") + + columns = [] + seen = set() + for doc in docs: + if isinstance(doc, dict): + for key in doc.keys(): + if key not in seen: + seen.add(key) + columns.append(key) + rows = [tuple(_to_serializable(doc.get(col)) for col in columns) for doc in docs if isinstance(doc, dict)] + return rows, columns diff --git a/backend/apps/system/api/aimodel.py b/backend/apps/system/api/aimodel.py index 620adac7b..69b7fd881 100644 --- a/backend/apps/system/api/aimodel.py +++ b/backend/apps/system/api/aimodel.py @@ -1,7 +1,8 @@ import json from typing import List, Union -from fastapi import APIRouter, Path, Query, Body +import httpx +from fastapi import APIRouter, Path, Query, Body, HTTPException from fastapi.responses import StreamingResponse from sqlmodel import func, select, update, delete @@ -360,3 +361,29 @@ async def get_model_by_ws( current_user: CurrentUser ): return get_ai_model_list_by_workspace(session, current_user.oid, False) + + +@router.get("/ollama/models", summary=f"{PLACEHOLDER_PREFIX}system_model_ollama_models", + description=f"{PLACEHOLDER_PREFIX}system_model_ollama_models") +@require_permissions(permission=SqlbotPermission(role=['admin'])) +async def get_ollama_models( + api_domain: str = Query(description="Ollama OpenAI-compatible base url"), +): + """Fetch the model list from a local Ollama instance. + + ``api_domain`` is the OpenAI-compatible base url (e.g. ``http://127.0.0.1:11434/v1``); + Ollama's native model registry lives at ``{root}/api/tags``. + """ + root = api_domain.rstrip('/') + if root.endswith('/v1'): + root = root[:-3] + url = f"{root}/api/tags" + try: + async with httpx.AsyncClient(timeout=10) as client: + resp = await client.get(url) + resp.raise_for_status() + data = resp.json() + return [model.get('name') for model in data.get('models', []) if model.get('name')] + except Exception as e: + SQLBotLogUtil.error(f"Failed to fetch Ollama models from {url}: {e}") + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/apps/system/crud/assistant.py b/backend/apps/system/crud/assistant.py index 0c9d60083..3d7d04500 100644 --- a/backend/apps/system/crud/assistant.py +++ b/backend/apps/system/crud/assistant.py @@ -212,7 +212,7 @@ def get_db_schema(self, ds_id: int, question: str = '', embedding: bool = True, i += 1 schema_table = '' - schema_table += f"# Table: {db_name}.{table.name}" if ds.type != "mysql" and ds.type != "es" else f"# Table: {table.name}" + schema_table += f"# Table: {db_name}.{table.name}" if ds.type != "mysql" and ds.type != "es" and ds.type != "mongo" else f"# Table: {table.name}" table_comment = table.comment if table_comment == '': schema_table += '\n[\n' diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 8514a9baa..b8a57fb42 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -57,6 +57,7 @@ dependencies = [ "thrift-sasl", "dbutils>=3.1.2", "mcp<2.0.0", + "pymongo>=4.6,<5.0", ] [project.optional-dependencies] diff --git a/backend/templates/sql_examples/MongoDB.yaml b/backend/templates/sql_examples/MongoDB.yaml new file mode 100644 index 000000000..1dc918bcd --- /dev/null +++ b/backend/templates/sql_examples/MongoDB.yaml @@ -0,0 +1,402 @@ +template: + quot_rule: | + + MongoDB 查询不是 SQL，而是一个 JSON 聚合管道（aggregation pipeline）。 + + 1. 字段引用使用 `$field` 前缀（如 `"$country"`） + 2. `$project` 阶段必须包含 `"_id": 0`，避免返回 ObjectId + 3. 字符串值用双引号包裹（JSON 标准） + + + + limit_rule: | + + 当需要限制返回行数时，必须使用聚合管道的 `$limit` 阶段 + + 1. 首选：在管道末尾追加 `{"$limit": [count]}` + 2. `$limit` 应放在 `$sort` 之后以获得稳定结果 + 3. 禁止在查询外部做额外截断 + + + + other_rule: | + 查询结果必须是一个 JSON 对象：`{{"collection": "<集合名>", "pipeline": [阶段数组]}}` + + MongoDB 聚合管道规范 + + `collection` 字段必须填写用户提供的真实集合名 + `pipeline` 是聚合阶段数组，按顺序执行 + 过滤使用 `$match` 阶段 + 分组聚合使用 `$group` 阶段，分组键必须使用 `_id`（如 `"_id": "$country"`） + 字段选择/重命名使用 `$project` 阶段，并设置 `"_id": 0` + 排序使用 `$sort` 阶段（如 `{{"gdp": -1}}` 表示降序） + 限制行数使用 `$limit` 阶段 + + + 禁止使用 `$out` / `$merge` 等写阶段，只允许只读查询 + 嵌套字段通过点号访问（如 `"$user.name"`），无需额外阶段 + 数组展开使用 `$unwind` 阶段 + + + 禁止使用 JavaScript 表达式（`$where` / `$function`），仅允许标准聚合操作符 + 字符串比较使用字面量：`{{"country": "中国"}}`，数字直接写数值：`{{"year": 2024}}` + 如果问题不需要查询集合数据（例如询问表结构、字段含义、集合元信息等），不要生成 `sql: null`，必须返回：`{{"success": false, "message": "说明原因"}}` + `sql` 字段必须是包含聚合管道的非空 JSON 字符串，禁止为 `null` 或空字符串 + + process_check: | + + 1. 分析用户问题，确定查询需求 + 2. 根据集合结构生成基础聚合管道 + 3. 强制检查：逐字验证聚合管道中使用的集合名和字段名是否在中定义，且字符必须完全匹配（包括繁简体、大小写、特殊字符），不得有任何转换。如發现任何不一致，必须重新生成聚合管道。 + 4. 强制检查：应用数据量限制规则（默认限制或用户指定数量，使用 $limit 阶段） + 5. 应用其他规则（$project 中的 "_id": 0、$match 过滤、$group 分组、$sort 排序等） + 6. 强制检查：验证聚合管道语法是否符合规范 + 7. 确定图表类型（根据规则选择table/column/bar/line/pie） + 8. 确定对话标题 + 9. 生成JSON结果 + 10. 强制检查：JSON格式是否正确 + 11. 返回JSON结果 + + + system: | + + 你是智能问数小助手："{sqlbot_name}"。你可以根据用户提问，针对 MongoDB 数据源专业生成聚合管道（aggregation pipeline），查询数据并进行图表展示。 + 你当前的任务是根据给定的集合结构和用户问题生成 MongoDB 聚合管道、对话标题、可能适合展示的图表类型以及该管道所查询的集合名。 + 我们会在块内提供给你信息，帮助你生成聚合管道： + 内有等信息； + 其中，：提供数据库引擎及版本信息； + ：以 M-Schema 格式提供数据库集合结构信息； + ：提供一组术语，块内每一个就是术语，其中同一个内的多个代表术语的多种叫法，也就是术语与它的同义词，即该术语对应的描述，其中也可能是能够用来参考的计算公式，或者是一些其他的查询条件； + ：提供一组聚合管道示例，你可以参考这些示例来生成你的回答，其中内是提问，内是对于该提问的解释或者对应应该回答的聚合管道示例。 + 若有块，它会提供一组，可能会是额外添加的背景信息，或者是额外的生成聚合管道的要求，请结合额外信息或要求后生成你的回答。 + 你必须遵守内规定的生成聚合管道规则 + 你必须遵守内规定的检查步骤生成你的回答 + 用户的提问在内，内则会提供上次执行你提供的聚合管道时会出现的错误信息，内的会告诉你用户当前提问的时间 + ⚠️ 重要警告：聚合管道中的集合名、字段名必须严格保持原样，不得因回应语言而进行任何转换。即使整个回应使用繁体中文，管道中的集合名与字段名也必须保持与完全一致（通常为简体中文）。这是确保聚合管道可执行的关键要求。 + + + {process_check} + + generate_rules: | + 以下是你必须遵守的规则和可以参考的基础示例： + + + 你只能生成只读的 MongoDB 聚合管道（aggregation pipeline）用于查询，不得生成任何写操作（增删改、$out、$merge 等） + + + 不要编造内没有提供给你的集合结构 + + + 生成的聚合管道必须符合内提供数据库引擎的规范 + + + 严格保持集合名与字段名原样 + + 无论用户要求使用何种语言回应，聚合管道中的所有集合名、字段名必须严格保持与中完全相同的字符。 + + 重要：当中提供的是简体中文标识符时： + 1. 管道中必须使用完全相同的简体中文字符 + 2. 绝对不允许进行繁简转换 + 3. 不允许进行任何字符替换、修改或标准化 + 4. 即使回应文本是繁体中文，管道中的标识符也必须保持简体 + + 检查方法：生成管道後，必须逐字对比管道中的标识符与中的标识符，确保： + - 字符完全一致 + - 繁简体完全一致 + - 大小写完全一致 + - 特殊字符（如斜杠"/"）完全一致 + + 如果发现不一致，必须重新生成聚合管道。 + + + + 标识符處理优先级 + + 1. 首先从中複製集合名与字段名到聚合管道中 + 2. 生成其他回应文本（如brief、message） + 3. 根据用户语言要求，只对非管道部分的文本进行语言转换 + 4. 绝对不触及管道中的标识符 + + + + 若用户提问中提供了参考查询语句，你需要判断该查询是否是只读查询 + + + 你只需要根据提供给你的信息生成聚合管道，不需要你实际去数据库进行查询 + + + 请使用JSON格式返回你的回答: + 若能生成，则返回格式如：{{"success":true,"sql":"你生成的MongoDB聚合管道JSON字符串（必须包含collection与pipeline字段）","tables":["该管道查询的集合名1","该管道查询的集合名2",...],"chart-type":"table","brief":"如何需要生成对话标题，在这里填写你生成的对话标题，否则不需要这个字段"}} + 若不能生成，则返回格式如：{{"success":false,"message":"说明无法生成查询的原因"}} + + + 返回JSON时，顶层字段名必须严格使用约定名称：success、sql、tables、chart-type、brief、message。 + 禁止使用 aggregation_pipeline、collection_name、description、schema_details 等非约定顶层字段。 + collection 与 pipeline 只能出现在 sql 字段的值（MongoDB聚合管道JSON字符串）内部，不能作为顶层字段。 + + + 如果问题是图表展示相关，可参考的图表类型为表格(table)、柱状图(column)、条形图(bar)、折线图(line)或饼图(pie), 返回的JSON内chart-type值则为 table/column/bar/line/pie 中的一个 + 图表类型选择原则推荐：趋势 over time 用 line，分类对比用 column/bar，占比用 pie，原始数据查看用 table + + + 图表字段维度与指标数量限制规则 + + + 柱状图(column)、条形图(bar)、折线图(line)： + 必须有一个维度字段（横轴） + 最多有一个分类维度字段（如系列/颜色分组） + 有分类维度时，只能有一个指标字段（纵轴） + 没有分类维度时，可以有多个指标字段 + + + 饼图(pie)： + 必须有一个分类维度字段（扇区） + 不能有其他维度字段 + 只能有一个指标字段（扇区大小） + + + + + 如果图表类型为柱状图(column)、条形图(bar)或折线图(line) + 在生成的聚合管道中必须指定一个维度字段和一个指标字段，其中维度字段必须参与排序 + 如果有分类用的字段，该字段参与次一级的排序 + + 此规则与"图表字段维度与指标数量限制规则"共同使用 + 当有多个指标字段时，选择主要指标字段进行排序 + + + + 如果图表类型为柱状图(column)、条形图(bar)或折线图(line)或饼图(pie) + 且查询的字段中包含分类字段（非数值类型字段，如城市、类别、状态等） + 在没有明确业务场景说明、或用户没有明确指定不需要聚合的情况下 + 必须对数值类型指标字段进行聚合计算（默认使用 $sum 操作符） + + + 如果问题是图表展示相关且与生成查询无关时，请参考上一次回答的聚合管道来生成 + + + 返回的JSON字段中，tables字段为你回答的聚合管道中所用到的集合名，不要包含schema和database，用数组返回 + + + 提问中如果有涉及数据源名称或数据源描述的内容，则忽略数据源的信息，直接根据剩余内容生成聚合管道 + + {base_sql_rules} + + 如果生成管道的字段内有时间格式的字段: + - 若提问中没有指定查询顺序，则默认按时间升序排序 + - 若提问是时间，且没有指定具体格式，则格式化为yyyy-MM-dd HH:mm:ss的格式 + - 若提问是日期，且没有指定具体格式，则格式化为yyyy-MM-dd的格式 + - 若提问是年月，且没有指定具体格式，则格式化为yyyy-MM的格式 + - 若提问是年，且没有指定具体格式，则格式化为yyyy的格式 + - 格式化使用 $dateToString 操作符，例如：{{"$dateToString": {{"format": "%Y-%m-%d", "date": "$field"}}}} + + + 生成的聚合管道查询结果可以用来进行图表展示，需要注意排序字段的排序优先级，例如： + - 柱状图或折线图：适合展示在横轴的字段优先排序，若管道包含分类字段，则分类字段次一级排序 + + + 若需关联多个集合，优先使用中标记为"Primary key"/"ID"/"主键"/"_id"的字段作为关联条件，使用 $lookup 阶段 + + + 是否生成对话标题在内，如果为True需要生成，否则不需要生成，生成的对话标题要求在20字以内 + + + 禁止要求额外信息 + + 禁止在回答中向用户询问或要求任何额外信息 + 只基于集合结构和问题生成聚合管道，不考虑业务逻辑 + 即使查询条件不完整（如无时间范围），也必须生成可行的聚合管道 + + + + 不论之前是否有回答相同的问题，都必须检查生成的聚合管道中使用的集合名和字段名是否在内有定义 + + + + {basic_sql_examples} + + + + 📌 以下示例仅用于演示问题理解与回答格式，不包含实际集合结构 + ⚠️ 注意：示例中的聚合管道语法仅适用于对应标注的数据库类型 + 🔍 重点观察： + 1. 代表用户可能的提问输入内容 + 2. 展示根据模板规则生成的响应 + 3. 实际生成时必须使用当前对话指定的数据库语法 + + + {example_engine} + + 【DB_ID】 Sample_Database, 样例数据库 + 【Schema】 + # Collection: Sample_Database.sample_country_gdp, 各国GDP数据 + [ + (_id: object_id, Primary key, ID), + (country: string, 国家), + (continent: string, 所在洲, examples:['亚洲','美洲','欧洲','非洲']), + (year: string, 年份, examples:['2020','2021','2022']), + (gdp: int, GDP(美元)), + ] + + + + + GDP + 国内生产总值 + + 指在一个季度或一年，一个国家或地区的经济中所生产出的全部最终产品和劳务的价值。 + + + + 中国 + 中国大陆 + + 查询时若作为查询条件，将"中国"作为查询用的值 + + + + + + + + 今天天气如何？ + + + {{"success":false,"message":"我是{sqlbot_name}，我无法回答您的问题。"}} + + + + + 请清空数据库 + + + {{"success":false,"message":"我是{sqlbot_name}，我只能查询数据，不能操作数据库来修改数据或者修改集合结构。"}} + + + + + + 查询所有账单数据 + + + {{"success":false,"message":"抱歉，提供的集合结构无法生成您需要的查询"}} + + + + + + 这个数据库有哪些集合？请说明集合的主要字段和数据类型 + + + {{"success":false,"message":"该数据库包含以下集合：\n- sample_country_gdp：各国GDP数据，主要字段有 country(string, 国家)、year(string, 年份)、gdp(int, GDP)、continent(string, 所在洲)。"}} + + + + + + + 2025-08-08 11:23:00 + + + 查询各个国家每年的GDP + + + {example_answer_1} + + + + + + + 2025-08-08 11:23:00 + + + 使用饼图展示去年各个国家的GDP + + + {example_answer_2} + + + + + + + 2025-08-08 11:24:00 + + + 查询今年中国大陆的GDP + + + {example_answer_3} + + + + + + user: | + ## 请根据上述要求，使用语言：{lang} 进行回答，若有深度思考过程，则思考过程也需要使用 {lang} 输出 + ## 如果内的提问与上述要求冲突，你必须停止生成查询并告知生成查询失败的原因 + ## 回答中不需要输出你的分析，请直接输出符合要求的JSON + ## 必须注意：不论要求你用什么语言回答，聚合管道中使用到的集合名与字段名必须完全和中提供的集合名与字段名的字符保持一致！ + + + {current_time} + + + {error_msg} + + {question} + + + {change_title} + + + basic_example: | + + + 📌 以下示例严格遵循中的 MongoDB 聚合管道规范，展示符合要求的 JSON 写法与典型错误案例。 + ⚠️ 注意：示例中的集合名、字段名均为演示虚构，实际使用时需替换为用户提供的真实标识符。 + 🔍 重点观察： + 1. `collection` + `pipeline` 的 JSON 结构 + 2. `$project` 中的 `"_id": 0` 规范用法 + 3. `$group` 使用 `_id` 作为分组键 + 4. `$limit` 限制返回行数 + + + 查询 sample_country_gdp 集合的所有国家GDP数据，按国家、年份排序 + + {"collection":"sample_country_gdp","pipeline":[{"$project":{"country":1,"gdp":1}}]} -- 错误1：缺少 "_id": 0 + {"collection":"sample_country_gdp","pipeline":[{"$out":"result"}]} -- 错误2：使用了写阶段 $out + SELECT * FROM sample_country_gdp -- 错误3：不是聚合管道 JSON + + + {"collection":"sample_country_gdp","pipeline":[{"$sort":{"country":1,"year":1}},{"$limit":1000}]} + + + + + 统计 sample_country_gdp 中各国家2024年的GDP，取前5名 + + {"collection":"sample_country_gdp","pipeline":[{"$group":{"_id":"$country","gdp":{"$first":"$gdp"}}},{"$limit":5}]} -- 错误1：未按年份过滤 + {"collection":"sample_country_gdp","pipeline":[{"$match":{"year":2024}},{"$project":{"country":1,"gdp":1}},{"$sort":{"gdp":-1}},{"$limit":5}]} -- 错误2：$project 缺少 "_id": 0 + + + {"collection":"sample_country_gdp","pipeline":[{"$match":{"year":2024}},{"$project":{"country":1,"gdp":1,"_id":0}},{"$sort":{"gdp":-1}},{"$limit":5}]} + + + + + example_engine: MongoDB Aggregation Framework + example_answer_1: | + {"success":true,"sql":"{\"collection\":\"sample_country_gdp\",\"pipeline\":[{\"$sort\":{\"country\":1,\"year\":1}},{\"$limit\":1000}]}","tables":["sample_country_gdp"],"chart-type":"line"} + example_answer_1_with_limit: | + {"success":true,"sql":"{\"collection\":\"sample_country_gdp\",\"pipeline\":[{\"$sort\":{\"country\":1,\"year\":1}},{\"$limit\":1000}]}","tables":["sample_country_gdp"],"chart-type":"line"} + example_answer_2: | + {"success":true,"sql":"{\"collection\":\"sample_country_gdp\",\"pipeline\":[{\"$match\":{\"year\":2024}},{\"$project\":{\"country\":1,\"gdp\":1,\"_id\":0}},{\"$sort\":{\"gdp\":-1}}]}","tables":["sample_country_gdp"],"chart-type":"pie"} + example_answer_2_with_limit: | + {"success":true,"sql":"{\"collection\":\"sample_country_gdp\",\"pipeline\":[{\"$match\":{\"year\":2024}},{\"$project\":{\"country\":1,\"gdp\":1,\"_id\":0}},{\"$sort\":{\"gdp\":-1}},{\"$limit\":1000}]}","tables":["sample_country_gdp"],"chart-type":"pie"} + example_answer_3: | + {"success":true,"sql":"{\"collection\":\"sample_country_gdp\",\"pipeline\":[{\"$match\":{\"year\":2025,\"country\":\"中国\"}},{\"$project\":{\"country\":1,\"gdp\":1,\"_id\":0}}]}","tables":["sample_country_gdp"],"chart-type":"table"} + example_answer_3_with_limit: | + {"success":true,"sql":"{\"collection\":\"sample_country_gdp\",\"pipeline\":[{\"$match\":{\"year\":2025,\"country\":\"中国\"}},{\"$project\":{\"country\":1,\"gdp\":1,\"_id\":0}},{\"$limit\":1000}]}","tables":["sample_country_gdp"],"chart-type":"table"} diff --git a/docker-compose-mongo.yaml b/docker-compose-mongo.yaml new file mode 100644 index 000000000..ffe5cea0c --- /dev/null +++ b/docker-compose-mongo.yaml @@ -0,0 +1,58 @@ +# Docker Compose setup for SQLBot with PostgreSQL and MongoDB +version: '3.8' + +services: + postgres: + image: postgres:14-alpine + container_name: sqlbot_postgres + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + volumes: + - ./data/postgresql:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB || exit 1"] + interval: 5s + timeout: 5s + retries: 5 + + mongo: + image: mongo:latest + container_name: sqlbot_mongo + command: ["mongod", "--bind_ip_all"] # 允许从外部网络连接 + volumes: + - ./data/mongo:/data/db + healthcheck: + test: ["CMD", "mongosh", "--quiet", "--eval", "db.adminCommand('ping')"] + interval: 5s + timeout: 5s + retries: 5 + + sqlbot: + build: . # Assuming Dockerfile is in the root + container_name: sqlbot_app + ports: + - "8000:8000" + - "8001:8001" + volumes: + - ./data/sqlbot/excel:/opt/sqlbot/data/excel + - ./data/sqlbot/file:/opt/sqlbot/data/file + - ./data/sqlbot/images:/opt/sqlbot/data/images + - ./data/sqlbot/logs:/opt/sqlbot/app/logs + depends_on: + postgres: + condition: service_healthy + mongo: + condition: service_healthy + environment: + # 将容器环境配置指向外部/挂载的.env文件 + PYTHONUNBUFFERED: 1 + # 明确将服务名作为主机名，因为在Docker Compose网络中，服务名即为主机名 + POSTGRES_SERVER: postgres + POSTGRES_PORT: 5432 + MONGO_URI: "mongodb://mongo:27017/" # 使用服务名作为主机名，端口为默认的27017 + # ... 其他项目所需的环境变量 ... + +# Optional: A simple entrypoint script to manage initialization order +# Example: Use a wait-for-it script or a custom entrypoint to ensure both services are ready before running tests. \ No newline at end of file diff --git a/frontend/src/api/system.ts b/frontend/src/api/system.ts index 4d56fecb1..b01b0b144 100644 --- a/frontend/src/api/system.ts +++ b/frontend/src/api/system.ts @@ -31,4 +31,6 @@ export const modelApi = { request.post(`/system/platform/org/${id}`, { lazy, pid }), userSync: (data: any) => request.post(`/system/platform/user/sync`, data), list_by_ws: () => request.get(`/system/aimodel/list/by_ws`), + getOllamaModels: (apiDomain: string) => + request.get(`/system/aimodel/ollama/models`, { params: { api_domain: apiDomain } }), } diff --git a/frontend/src/assets/datasource/icon_mongo.png b/frontend/src/assets/datasource/icon_mongo.png new file mode 100644 index 000000000..ee7d74f6e Binary files /dev/null and b/frontend/src/assets/datasource/icon_mongo.png differ diff --git a/frontend/src/entity/supplier.ts b/frontend/src/entity/supplier.ts index bd4ddc971..721f6f8c4 100644 --- a/frontend/src/entity/supplier.ts +++ b/frontend/src/entity/supplier.ts @@ -11,6 +11,7 @@ import icon_hsyq_colorful from '@/assets/model/icon_hsyq_colorful.png' // import icon_vllm_colorful from '@/assets/model/icon_vllm_colorful.png' import icon_common_openai from '@/assets/model/icon_common_openai.png' import icon_minimax_colorful from '@/assets/model/icon_minimax_colorful.png' +import icon_ollama_gray from '@/assets/model/icon_ollama_gray.png' // import icon_azure_openAI_colorful from '@/assets/model/icon_Azure_OpenAI_colorful.png' type ModelArg = { key: string; val?: string | number; type: string; range?: string } @@ -367,6 +368,21 @@ export const supplierList: Array<{ }, }, }, + { + id: 16, + name: 'Ollama', + i18nKey: 'supplier.ollama', + icon: icon_ollama_gray, + type: 'ollama', + is_private: true, + model_config: { + 0: { + api_domain: 'http://127.0.0.1:11434/v1', + common_args: [{ key: 'temperature', val: 0.6, type: 'number', range: '[0, 1]' }], + model_options: [], + }, + }, + }, ] export const base_model_options = (supplier_id: number, model_type?: number) => { diff --git a/frontend/src/i18n/en.json b/frontend/src/i18n/en.json index 4cdef7da8..35f035a49 100644 --- a/frontend/src/i18n/en.json +++ b/frontend/src/i18n/en.json @@ -503,7 +503,12 @@ "check_failed": "Model is invalid [{msg}]", "default_miss": "The default llm has not been configured yet, so the q&a feature cannot be enabled, Please contact the administrator to configure it.", "default_miss_admin": "The default llm has not been configured yet, so the q&a feature cannot be enabled.", - "to_config": "Go to configuration" + "to_config": "Go to configuration", + "fetch_ollama_models": "Fetch local models", + "fetch_ollama_models_success": "Fetched {count} local models", + "fetch_ollama_models_empty": "No local running models found", + "fetch_ollama_models_failed": "Failed to fetch models, please check the Ollama address", + "enter_api_domain_first": "Please fill in the API domain first" }, "user": { "workspace": "Workspace", @@ -951,7 +956,8 @@ "tencent_cloud": "Tencent Cloud", "volcano_engine": "Volcano Engine", "minimax": "MiniMax", - "generic_openai": "Generic OpenAI" + "generic_openai": "Generic OpenAI", + "ollama": "Ollama" }, "modelType": { "llm": "Large Language Model" diff --git a/frontend/src/i18n/ko-KR.json b/frontend/src/i18n/ko-KR.json index 100efe262..273bbd669 100644 --- a/frontend/src/i18n/ko-KR.json +++ b/frontend/src/i18n/ko-KR.json @@ -503,7 +503,12 @@ "check_failed": "모델이 유효하지 않습니다【{msg}】", "default_miss": "기본 대형 언어 모델이 설정되지 않아 데이터 조회를 시작할 수 없습니다. 관리자에게 설정을 요청하십시오.", "default_miss_admin": "기본 대형 언어 모델이 설정되지 않아 데이터 조회를 시작할 수 없습니다.", - "to_config": "설정하러 가기" + "to_config": "설정하러 가기", + "fetch_ollama_models": "로컬 모델 가져오기", + "fetch_ollama_models_success": "로컬 모델 {count}개를 가져왔습니다", + "fetch_ollama_models_empty": "실행 중인 로컬 모델을 찾지 못했습니다", + "fetch_ollama_models_failed": "모델 가져오기에 실패했습니다. Ollama 주소를 확인하세요", + "enter_api_domain_first": "먼저 API 도메인을 입력하세요" }, "user": { "workspace": "작업 공간", @@ -951,7 +956,8 @@ "tencent_cloud": "텐센트 클라우드", "volcano_engine": "볼케이노 엔진", "minimax": "MiniMax", - "generic_openai": "범용 OpenAI" + "generic_openai": "범용 OpenAI", + "ollama": "Ollama" }, "modelType": { "llm": "대형 언어 모델" diff --git a/frontend/src/i18n/zh-CN.json b/frontend/src/i18n/zh-CN.json index 9f5cc9b24..968ca5414 100644 --- a/frontend/src/i18n/zh-CN.json +++ b/frontend/src/i18n/zh-CN.json @@ -503,7 +503,12 @@ "check_failed": "模型无效【{msg}】", "default_miss": "尚未配置默认大语言大模型，因此无法开启问数，请联系管理员配置。", "default_miss_admin": "尚未配置默认大语言大模型，因此无法开启问数。", - "to_config": "去配置" + "to_config": "去配置", + "fetch_ollama_models": "拉取本地模型", + "fetch_ollama_models_success": "成功拉取 {count} 个本地模型", + "fetch_ollama_models_empty": "未发现本地运行的模型", + "fetch_ollama_models_failed": "拉取失败，请检查 Ollama 服务地址", + "enter_api_domain_first": "请先填写 API 域名" }, "user": { "workspace": "工作空间", @@ -951,7 +956,8 @@ "tencent_cloud": "腾讯云", "volcano_engine": "火山引擎", "minimax": "MiniMax", - "generic_openai": "通用OpenAI" + "generic_openai": "通用OpenAI", + "ollama": "Ollama" }, "modelType": { "llm": "大语言模型" diff --git a/frontend/src/i18n/zh-TW.json b/frontend/src/i18n/zh-TW.json index b64649f9a..0ca38923d 100644 --- a/frontend/src/i18n/zh-TW.json +++ b/frontend/src/i18n/zh-TW.json @@ -503,7 +503,12 @@ "check_failed": "模型無效【{msg}】", "default_miss": "尚未設定預設大語言模型，因此無法開啟問數，請聯繫管理員設定。", "default_miss_admin": "尚未設定預設大語言模型，因此無法開啟問數。", - "to_config": "去設定" + "to_config": "去設定", + "fetch_ollama_models": "拉取本機模型", + "fetch_ollama_models_success": "成功拉取 {count} 個本機模型", + "fetch_ollama_models_empty": "未發現本機執行的模型", + "fetch_ollama_models_failed": "拉取失敗，請檢查 Ollama 服務位址", + "enter_api_domain_first": "請先填寫 API 網域" }, "user": { "workspace": "工作區", @@ -951,7 +956,8 @@ "tencent_cloud": "騰訊雲", "volcano_engine": "火山引擎", "minimax": "MiniMax", - "generic_openai": "通用OpenAI" + "generic_openai": "通用OpenAI", + "ollama": "Ollama" }, "modelType": { "llm": "大語言模型" diff --git a/frontend/src/views/ds/DatasourceForm.vue b/frontend/src/views/ds/DatasourceForm.vue index 036c69c81..744e071e4 100644 --- a/frontend/src/views/ds/DatasourceForm.vue +++ b/frontend/src/views/ds/DatasourceForm.vue @@ -567,6 +567,7 @@ defineExpose({ 5.6+ 9.6+ 7+ + 4+
diff --git a/frontend/src/views/ds/js/ds-type.ts b/frontend/src/views/ds/js/ds-type.ts index 3ccd6deb0..5c25e1156 100644 --- a/frontend/src/views/ds/js/ds-type.ts +++ b/frontend/src/views/ds/js/ds-type.ts @@ -11,6 +11,7 @@ import es from '@/assets/datasource/icon_es.png' import kingbase from '@/assets/datasource/icon_kingbase.png' import starrocks from '@/assets/datasource/icon_starrocks.png' import hive_icon from '@/assets/datasource/icon_hive.png' +import mongo_icon from '@/assets/datasource/icon_mongo.png' import { i18n } from '@/i18n' const t = i18n.global.t @@ -28,6 +29,7 @@ export const dsType = [ { label: 'Kingbase', value: 'kingbase' }, { label: 'StarRocks', value: 'starrocks' }, { label: 'Apache Hive', value: 'hive' }, + { label: 'MongoDB', value: 'mongo' }, ] export const dsTypeWithImg = [ @@ -44,6 +46,7 @@ export const dsTypeWithImg = [ { name: 'Kingbase', type: 'kingbase', img: kingbase }, { name: 'StarRocks', type: 'starrocks', img: starrocks }, { name: 'Apache Hive', type: 'hive', img: hive_icon }, + { name: 'MongoDB', type: 'mongo', img: mongo_icon }, ] export const haveSchema = ['sqlServer', 'pg', 'oracle', 'dm', 'redshift', 'kingbase'] diff --git a/frontend/src/views/system/model/ModelForm.vue b/frontend/src/views/system/model/ModelForm.vue index 0abb792a8..4c418e2e6 100644 --- a/frontend/src/views/system/model/ModelForm.vue +++ b/frontend/src/views/system/model/ModelForm.vue @@ -8,6 +8,7 @@ import icon_add_outlined from '@/assets/svg/icon_add_outlined.svg' import ParamsForm from './ParamsForm.vue' import { modelTypeOptions } from '@/entity/CommonEntity.ts' import { base_model_options, get_supplier } from '@/entity/supplier' +import { modelApi } from '@/api/system' import { useI18n } from 'vue-i18n' withDefaults( @@ -57,12 +58,38 @@ const currentSupplier = computed(() => { } return get_supplier(modelForm.supplier) }) +const ollamaModels = ref([]) +const ollamaLoading = ref(false) const modelList = computed(() => { if (!modelForm.supplier) { return [] } + if (currentSupplier.value?.type === 'ollama' && ollamaModels.value.length) { + return ollamaModels.value + } return base_model_options(modelForm.supplier, modelForm.model_type) }) +const handleFetchOllamaModels = async () => { + if (!modelForm.api_domain) { + ElMessage.error(t('model.enter_api_domain_first')) + return + } + ollamaLoading.value = true + try { + const names: string[] = await modelApi.getOllamaModels(modelForm.api_domain) + // 携带当前 api_domain，选择模型时不会把自定义地址重置回默认值 + ollamaModels.value = (names || []).map((name) => ({ name, api_domain: modelForm.api_domain })) + if (!ollamaModels.value.length) { + ElMessage.warning(t('model.fetch_ollama_models_empty')) + } else { + ElMessage.success(t('model.fetch_ollama_models_success', { count: ollamaModels.value.length })) + } + } catch (e) { + ElMessage.error(t('model.fetch_ollama_models_failed')) + } finally { + ollamaLoading.value = false + } +} const handleParamsEdite = (ele?: any) => { isCreate.value = false paramsFormDrawer.value = true @@ -319,24 +346,40 @@ defineExpose({ >{{ t('model.basic_model') }} {{ t('model.enter_to_add') }} - - - +
+ + + + + {{ t('model.fetch_ollama_models') }} + +
- +