diff --git a/examples/bash-comparemodels/README.md b/examples/bash-comparemodels/README.md new file mode 100644 index 00000000..91499255 --- /dev/null +++ b/examples/bash-comparemodels/README.md @@ -0,0 +1,10 @@ +# Bash Shell examples + +When calling `ollama`, you can pass it a file to run all the prompts in the file, one after the other: + +`ollama run llama2 < sourcequestions.txt` + +This concept is used in the following example. + +## Compare Models +`comparemodels.sh` is a script that runs all the questions in `sourcequestions.txt` using any 4 models you choose that you have already pulled from the Ollama library or have created locally. diff --git a/examples/bash-comparemodels/comparemodels.sh b/examples/bash-comparemodels/comparemodels.sh new file mode 100755 index 00000000..1ce249a6 --- /dev/null +++ b/examples/bash-comparemodels/comparemodels.sh @@ -0,0 +1,64 @@ +#! /usr/bin/env bash +# Compare multiple models by running them with the same questions + +NUMBEROFCHOICES=4 +SELECTIONS=() +declare -a SUMS=() + +# Get the list of models +CHOICES=$(ollama list | awk '{print $1}') + +# Select which models to run as a comparison +echo "Select $NUMBEROFCHOICES models to compare:" +select ITEM in $CHOICES; do + if [[ -n $ITEM ]]; then + echo "You have selected $ITEM" + SELECTIONS+=("$ITEM") + ((COUNT++)) + if [[ $COUNT -eq $NUMBEROFCHOICES ]]; then + break + fi + else + echo "Invalid selection" + fi +done + +# Loop through each of the selected models +for ITEM in "${SELECTIONS[@]}"; do + echo "--------------------------------------------------------------" + echo "Loading the model $ITEM into memory" + ollama run "$ITEM" "" + echo "--------------------------------------------------------------" + echo "Running the questions through the model $ITEM" + COMMAND_OUTPUT=$(ollama run "$ITEM" --verbose < sourcequestions.txt 2>&1| tee /dev/stderr) + + # eval duration is sometimes listed in seconds and sometimes in milliseconds. + # Add up the values for each model + SUM=$(echo "$COMMAND_OUTPUT" | awk ' + /eval duration:/ { + value = $3 + if (index(value, "ms") > 0) { + gsub("ms", "", value) + value /= 1000 + } else { + gsub("s", "", value) + } + sum += value + } + END { print sum }') + + + SUMS+=("All questions for $ITEM completed in $SUM seconds") +done + +echo "" +echo "--------------------------------------------------------------" +echo -e "Sums of eval durations for each run:" +for val in "${SUMS[@]}"; do + echo "$val" +done + +echo "--------------------------------------------------------------" +echo "Comparison complete. Now you can decide" +echo "which model is best." +echo "--------------------------------------------------------------" \ No newline at end of file diff --git a/examples/bash-comparemodels/sourcequestions.txt b/examples/bash-comparemodels/sourcequestions.txt new file mode 100644 index 00000000..90004c07 --- /dev/null +++ b/examples/bash-comparemodels/sourcequestions.txt @@ -0,0 +1,7 @@ +Why is the sky blue +What is a black hole +Explain the big bang theory like I am 5? +What is the quickest way to win a game of Monopoly with 3 others? +Why does a vacuum bottle keep my coffee hot and my milkshake cold? +What is the difference between a meteor, a meteorite, and a meteoroid? +Create an array with 5 items and print to the console. Do this in Python, C#, Typescript, and Rust. \ No newline at end of file