From 64c4b863f4ce248adad5580849c27af987ae2833 Mon Sep 17 00:00:00 2001 From: Xiaorui Xue Date: Fri, 5 Apr 2024 21:09:14 +0800 Subject: [PATCH 1/2] added ollama support --- app/services/llm.py | 7 +++++++ config.example.toml | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/app/services/llm.py b/app/services/llm.py index 84ada77..ea5bc30 100644 --- a/app/services/llm.py +++ b/app/services/llm.py @@ -27,6 +27,13 @@ def _generate_response(prompt: str) -> str: api_key = config.app.get("moonshot_api_key") model_name = config.app.get("moonshot_model_name") base_url = "https://api.moonshot.cn/v1" + elif llm_provider == "ollama": + # api_key = config.app.get("openai_api_key") + api_key = "ollama" # any string works but you are required to have one + model_name = config.app.get("ollama_model_name") + base_url = config.app.get("ollama_base_url", "") + if not base_url: + base_url = "http://localhost:11434/v1" elif llm_provider == "openai": api_key = config.app.get("openai_api_key") model_name = config.app.get("openai_model_name") diff --git a/config.example.toml b/config.example.toml index fd4adba..936c060 100644 --- a/config.example.toml +++ b/config.example.toml @@ -19,6 +19,12 @@ # gemini llm_provider="openai" + ########## Ollama Settings + # No need to set it unless you want to use your own proxy + ollama_base_url = "" + # Check your available models at https://platform.openai.com/account/limits + ollama_model_name = "qwen:0.5b" + ########## OpenAI API Key # Get your API key at https://platform.openai.com/api-keys openai_api_key = "" From 045c8521a452ed5eff7deb6ad592e9a6dc11a581 Mon Sep 17 00:00:00 2001 From: Xiaorui Xue Date: Fri, 5 Apr 2024 21:11:45 +0800 Subject: [PATCH 2/2] small changes to make config file better --- config.example.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.example.toml b/config.example.toml index 936c060..634706f 100644 --- a/config.example.toml +++ b/config.example.toml @@ -22,8 +22,8 @@ ########## Ollama Settings # No need to set it unless you want to use your own proxy ollama_base_url = "" - # Check your available models at https://platform.openai.com/account/limits - ollama_model_name = "qwen:0.5b" + # Check your available models at https://ollama.com/library + ollama_model_name = "" ########## OpenAI API Key # Get your API key at https://platform.openai.com/api-keys