From 43027789dccef1c0430d178e14bf361ed8ef4842 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Wed, 6 Dec 2023 14:35:58 -0800 Subject: [PATCH 1/6] Simple chat example Signed-off-by: Matt Williams --- examples/python-simplechat/client.py | 48 ++++++++++++++++++++++++++++ examples/python-simplechat/readme.md | 24 ++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 examples/python-simplechat/client.py create mode 100644 examples/python-simplechat/readme.md diff --git a/examples/python-simplechat/client.py b/examples/python-simplechat/client.py new file mode 100644 index 00000000..63d2f141 --- /dev/null +++ b/examples/python-simplechat/client.py @@ -0,0 +1,48 @@ +import json +import requests + +# NOTE: ollama must be running for this to work, start the ollama app or run `ollama serve` +model = "llama2" # TODO: update this for whatever model you wish to use + + +def chat(messages): + r = requests.post( + "http://0.0.0.0:11434/api/chat", + json={"model": model, "messages": messages, "stream": True}, + ) + r.raise_for_status() + output = "" + + for line in r.iter_lines(): + body = json.loads(line) + if body.get("done") is False: + message = body.get("message", "") + content = message.get("content", "") + output += content + # the response streams one token at a time, print that as we receive it + print(content, end="", flush=True) + + if "error" in body: + raise Exception(body["error"]) + + if body.get("done", False): + message["content"] = output + return message + + +def main(): + messages = [] + context = ( + [] + ) # the context stores a conversation history, you can use this to make the model more context aware + while True: + user_input = input("Enter a prompt: ") + print() + messages.append({"role": "user", "content": user_input}) + message = chat(messages) + messages.append(message) + print("\n\n") + + +if __name__ == "__main__": + main() diff --git a/examples/python-simplechat/readme.md b/examples/python-simplechat/readme.md new file mode 100644 index 00000000..e9fbc245 --- /dev/null +++ b/examples/python-simplechat/readme.md @@ -0,0 +1,24 @@ +# Simple Chat Example + +The **chat** endpoint is one of two ways to generate text from an LLM with Ollama. At a high level you provide the endpoint an array of objects with a role and content specified. Then with each output and prompt, you add more of those role/content objects, which builds up the memory. + +## Review the Code + +You can see in the **chat** function that actually calling the endpoint is done simply with: + +```python +r = requests.post( + "http://0.0.0.0:11434/api/chat", + json={"model": model, "messages": messages, "stream": True}, +) +``` + +With the **generate** endpoint, you need to provide a `prompt`. But with **chat**, you provide `messages`. And the resulting stream of responses includes a `message` object with a `content` field. + +The final JSON object doesn't provide the full content, so you will need to build the content yourself. + +In the **main** function, we collect `user_input` and add it as a message to our messages and that is passed to the chat function. And the output is added as another message. + +## Next Steps + +In this example, all generations are kept. You might want to experiment with summarizing everything older than 10 conversations to enable longer history with less context being used. \ No newline at end of file From 51131cc6e270d2b52e8e348abd275a618d5f2640 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Wed, 6 Dec 2023 15:30:10 -0800 Subject: [PATCH 2/6] Update examples/python-simplechat/client.py Co-authored-by: Bruce MacDonald --- examples/python-simplechat/client.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/python-simplechat/client.py b/examples/python-simplechat/client.py index 63d2f141..3f1d0bb1 100644 --- a/examples/python-simplechat/client.py +++ b/examples/python-simplechat/client.py @@ -32,8 +32,6 @@ def chat(messages): def main(): messages = [] - context = ( - [] ) # the context stores a conversation history, you can use this to make the model more context aware while True: user_input = input("Enter a prompt: ") From f337642e94fe44514096f776bf03834d591a846b Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Wed, 6 Dec 2023 15:30:35 -0800 Subject: [PATCH 3/6] Update examples/python-simplechat/readme.md Co-authored-by: Bruce MacDonald --- examples/python-simplechat/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python-simplechat/readme.md b/examples/python-simplechat/readme.md index e9fbc245..6f4aade5 100644 --- a/examples/python-simplechat/readme.md +++ b/examples/python-simplechat/readme.md @@ -1,6 +1,6 @@ # Simple Chat Example -The **chat** endpoint is one of two ways to generate text from an LLM with Ollama. At a high level you provide the endpoint an array of objects with a role and content specified. Then with each output and prompt, you add more of those role/content objects, which builds up the memory. +The **chat** endpoint is one of two ways to generate text from an LLM with Ollama. At a high level you provide the endpoint an array of objects with a role and content specified. Then with each output and prompt, you add more of those role/content objects, which builds up the history. ## Review the Code From aec742b6d21056a4dbb45c3b82c6c3552b47b9bc Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Wed, 6 Dec 2023 15:30:45 -0800 Subject: [PATCH 4/6] Update examples/python-simplechat/readme.md Co-authored-by: Bruce MacDonald --- examples/python-simplechat/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python-simplechat/readme.md b/examples/python-simplechat/readme.md index 6f4aade5..e4543c6c 100644 --- a/examples/python-simplechat/readme.md +++ b/examples/python-simplechat/readme.md @@ -17,7 +17,7 @@ With the **generate** endpoint, you need to provide a `prompt`. But with **chat* The final JSON object doesn't provide the full content, so you will need to build the content yourself. -In the **main** function, we collect `user_input` and add it as a message to our messages and that is passed to the chat function. And the output is added as another message. +In the **main** function, we collect `user_input` and add it as a message to our messages and that is passed to the chat function. When the LLM is done responding the output is added as another message. ## Next Steps From 4a02945a158dd5d1b5e3f1e2e3b8b4beae6b9aba Mon Sep 17 00:00:00 2001 From: Bruce MacDonald Date: Wed, 6 Dec 2023 18:36:45 -0500 Subject: [PATCH 5/6] Update examples/python-simplechat/client.py --- examples/python-simplechat/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/python-simplechat/client.py b/examples/python-simplechat/client.py index 3f1d0bb1..b72804bf 100644 --- a/examples/python-simplechat/client.py +++ b/examples/python-simplechat/client.py @@ -32,7 +32,6 @@ def chat(messages): def main(): messages = [] - ) # the context stores a conversation history, you can use this to make the model more context aware while True: user_input = input("Enter a prompt: ") print() From 76bc4d0458c298435c928cad0e46dcb9d95871b4 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Wed, 6 Dec 2023 15:44:40 -0800 Subject: [PATCH 6/6] Cleanup as per Bruce Signed-off-by: Matt Williams --- examples/python-simplechat/client.py | 6 +++--- examples/python-simplechat/readme.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/python-simplechat/client.py b/examples/python-simplechat/client.py index 3f1d0bb1..3c480f97 100644 --- a/examples/python-simplechat/client.py +++ b/examples/python-simplechat/client.py @@ -15,6 +15,8 @@ def chat(messages): for line in r.iter_lines(): body = json.loads(line) + if "error" in body: + raise Exception(body["error"]) if body.get("done") is False: message = body.get("message", "") content = message.get("content", "") @@ -22,8 +24,6 @@ def chat(messages): # the response streams one token at a time, print that as we receive it print(content, end="", flush=True) - if "error" in body: - raise Exception(body["error"]) if body.get("done", False): message["content"] = output @@ -32,7 +32,7 @@ def chat(messages): def main(): messages = [] - ) # the context stores a conversation history, you can use this to make the model more context aware + while True: user_input = input("Enter a prompt: ") print() diff --git a/examples/python-simplechat/readme.md b/examples/python-simplechat/readme.md index e4543c6c..abbdfe7e 100644 --- a/examples/python-simplechat/readme.md +++ b/examples/python-simplechat/readme.md @@ -21,4 +21,4 @@ In the **main** function, we collect `user_input` and add it as a message to our ## Next Steps -In this example, all generations are kept. You might want to experiment with summarizing everything older than 10 conversations to enable longer history with less context being used. \ No newline at end of file +In this example, all generations are kept. You might want to experiment with summarizing everything older than 10 conversations to enable longer history with less context being used.