diff options
Diffstat (limited to 'main.py')
| -rw-r--r-- | main.py | 18 |
1 files changed, 7 insertions, 11 deletions
@@ -1,20 +1,15 @@ #!/usr/bin/env python3 import asyncio import json +import re # thank you claude import ollama import websockets from websockets.server import serve from datetime import datetime -chatmodel = "hf.co/Orenguteng/Llama-3.1-8B-Lexi-Uncensored-V2-GGUF:Llama-3.1-8B-Lexi-Uncensored_V2_Q4.gguf" +chatmodel = "hf.co/Stetsoncody/Llama-3.2-3B-Instruct-abliterated-GGUF:Q4_K_M" embedmodel = 'hf.co/CompendiumLabs/bge-base-en-v1.5-gguf' -# these are test models just ignore -#chatmodel = 'hf.co/bartowski/Llama-3.2-1B-Instruct-GGUF' -#chatmodel = 'hf.co/VibeStudio/Nidum-Llama-3.2-3B-Uncensored-GGUF' -#chatmodel = "faggotchan" -#chatmodel = "qwen3.5:4b" - NUM_THREADS = 6 # hardcoded since threads.txt is gone, change if needed def times() -> str: @@ -160,16 +155,17 @@ async def ollamachat(websocket): model=chatmodel, messages=messages, options={ - "temperature": 0.8, - "top_p": 0.9, + "temperature": 0.83, + "top_p": 0.9 "top_k": 40, "repeat_penalty": 1.1, - "num_predict": 4096, + "num_predict": 1024, "num_thread": NUM_THREADS, } ) - bot_reply = response["message"]["content"].strip() + # strip <think> blocks in case the model ignores think=False, thank you claude + bot_reply = re.sub(r'<think>.*?</think>', '', response["message"]["content"], flags=re.DOTALL).strip() print(messages) print(f"[{times()}] reply: {bot_reply}\n") |