diff --git a/server/images.go b/server/images.go
index 6f72f0cf..438e5b8d 100644
--- a/server/images.go
+++ b/server/images.go
@@ -69,10 +69,14 @@ func (m *Model) Prompt(request api.GenerateRequest) (string, error) {
 	}
 
 	vars.First = len(request.Context) == 0
-	vars.System = m.System
+
 	vars.Prompt = request.Prompt
 	vars.Context = request.Context
 
+	if len(vars.Context) == 0 {
+		vars.System = m.System
+	}
+
 	if request.System != "" {
 		vars.System = request.System
 	}