From e89881104ad534552de7e3bb9d3f50adc9c0602a Mon Sep 17 00:00:00 2001 From: jiangquan Date: Tue, 20 Jun 2023 01:56:32 +0800 Subject: [PATCH 1/2] Change to Vicuna 1.1 template --- llama_cpp/llama.py | 7 ++++--- start_server.bat | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 start_server.bat diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index a0b2030174..3f6bb6e3ec 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1401,11 +1401,12 @@ def create_chat_completion( stop if isinstance(stop, list) else [stop] if isinstance(stop, str) else [] ) chat_history = "".join( - f'### {"Human" if message["role"] == "user" else "Assistant"}:{message["content"]}' + f'{message["role"].upper()}:{message["content"]}\n' for message in messages ) - PROMPT = chat_history + "### Assistant:" - PROMPT_STOP = ["### Assistant:", "### Human:"] + PROMPT = chat_history + "\nASSISTANT:" + print(f'The prompt is:\n{PROMPT}') + PROMPT_STOP = ["ASSISTANT:", "USER:"] completion_or_chunks = self( prompt=PROMPT, stop=PROMPT_STOP + stop, diff --git a/start_server.bat b/start_server.bat new file mode 100644 index 0000000000..c08a3481e4 --- /dev/null +++ b/start_server.bat @@ -0,0 +1,2 @@ +call .\venv\Scripts\activate.bat +python -m llama_cpp.server --model ..\models\wizardlm-7b-v1.0-uncensored.ggmlv3.q5_K_M.bin --port 8300 --host 0.0.0.0 --model_alias gpt-3.5-turbo --n_threads 16 --n_gpu_layers 50 \ No newline at end of file From 48801376a6b602a60aee9ae36940724197a8aa23 Mon Sep 17 00:00:00 2001 From: JohnJiang Date: Tue, 20 Jun 2023 18:11:43 +0800 Subject: [PATCH 2/2] script --- start_server.sh | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 start_server.sh diff --git a/start_server.sh b/start_server.sh new file mode 100644 index 0000000000..4f51314aa0 --- /dev/null +++ b/start_server.sh @@ -0,0 +1,8 @@ +# python -m venv ./venv +# git submodule update --init --recursive +# pip install scikit-build +# LLAMA_METAL=1 pip install -e .\[server\] +# LLAMA_METAL=1 python setup.py develop + +source ./venv/bin/activate +python -m llama_cpp.server --model ../models/wizardlm-7b-v1.0-uncensored.ggmlv3.q5_K_M.bin --model_alias gpt-3.5-turbo --port 8300 --host 0.0.0.0 --use_mlock 1 --use_mmap 1 --cache 1 --n_threads 4 \ No newline at end of file