diff --git a/Cargo.lock b/Cargo.lock index f644dfd..487034a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1021,7 +1021,7 @@ checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" [[package]] name = "llm-ls" -version = "0.1.0" +version = "0.1.1" dependencies = [ "home", "reqwest", diff --git a/src/language_comments.rs b/src/language_comments.rs deleted file mode 100644 index 41c521a..0000000 --- a/src/language_comments.rs +++ /dev/null @@ -1,413 +0,0 @@ -use crate::LanguageComment; -use std::collections::HashMap; - -pub fn build_language_comments() -> HashMap { - HashMap::from([ - ( - "abap".to_owned(), - LanguageComment { - open: "*".to_owned(), - close: None, - }, - ), - ( - "bat".to_owned(), - LanguageComment { - open: "REM".to_owned(), - close: None, - }, - ), - ( - "bibtex".to_owned(), - LanguageComment { - open: "%".to_owned(), - close: None, - }, - ), - ( - "clojure".to_owned(), - LanguageComment { - open: ";;".to_owned(), - close: None, - }, - ), - ( - "coffeescript".to_owned(), - LanguageComment { - open: "#".to_owned(), - close: None, - }, - ), - ( - "c".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "cpp".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "csharp".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "css".to_owned(), - LanguageComment { - open: "/*".to_owned(), - close: Some("*/".to_owned()), - }, - ), - ( - "diff".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "dart".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "dockerfile".to_owned(), - LanguageComment { - open: "#".to_owned(), - close: None, - }, - ), - ( - "elixir".to_owned(), - LanguageComment { - open: "#".to_owned(), - close: None, - }, - ), - ( - "erlang".to_owned(), - LanguageComment { - open: "%".to_owned(), - close: None, - }, - ), - ( - "fsharp".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "git-commit".to_owned(), - LanguageComment { - open: "#".to_owned(), - close: None, - }, - ), - ( - "git-rebase".to_owned(), - LanguageComment { - open: "#".to_owned(), - close: None, - }, - ), - ( - "go".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "groovy".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "handlebars".to_owned(), - LanguageComment { - open: "{{!--".to_owned(), - close: Some("--}}".to_owned()), - }, - ), - ( - "html".to_owned(), - LanguageComment { - open: "".to_owned()), - }, - ), - ( - "ini".to_owned(), - LanguageComment { - open: ";".to_owned(), - close: None, - }, - ), - ( - "java".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "javascript".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "javascriptreact".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "json".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "latex".to_owned(), - LanguageComment { - open: "%".to_owned(), - close: None, - }, - ), - ( - "less".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "lua".to_owned(), - LanguageComment { - open: "--".to_owned(), - close: None, - }, - ), - ( - "makefile".to_owned(), - LanguageComment { - open: "#".to_owned(), - close: None, - }, - ), - ( - "markdown".to_owned(), - LanguageComment { - open: "".to_owned()), - }, - ), - ( - "objective-c".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "objective-cpp".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "perl".to_owned(), - LanguageComment { - open: "#".to_owned(), - close: None, - }, - ), - ( - "perl6".to_owned(), - LanguageComment { - open: "#".to_owned(), - close: None, - }, - ), - ( - "php".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "powershell".to_owned(), - LanguageComment { - open: "#".to_owned(), - close: None, - }, - ), - ( - "jade".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "python".to_owned(), - LanguageComment { - open: "#".to_owned(), - close: None, - }, - ), - ( - "r".to_owned(), - LanguageComment { - open: "#".to_owned(), - close: None, - }, - ), - ( - "razor".to_owned(), - LanguageComment { - open: "@*".to_owned(), - close: Some("*@".to_owned()), - }, - ), - ( - "ruby".to_owned(), - LanguageComment { - open: "#".to_owned(), - close: None, - }, - ), - ( - "rust".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "scss".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "sass".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "scala".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "shaderlab".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "shellscript".to_owned(), - LanguageComment { - open: "#".to_owned(), - close: None, - }, - ), - ( - "sql".to_owned(), - LanguageComment { - open: "--".to_owned(), - close: None, - }, - ), - ( - "swift".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "toml".to_owned(), - LanguageComment { - open: "#".to_owned(), - close: None, - }, - ), - ( - "typescript".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "typescriptreact".to_owned(), - LanguageComment { - open: "//".to_owned(), - close: None, - }, - ), - ( - "tex".to_owned(), - LanguageComment { - open: "%".to_owned(), - close: None, - }, - ), - ( - "vb".to_owned(), - LanguageComment { - open: "'".to_owned(), - close: None, - }, - ), - ( - "xml".to_owned(), - LanguageComment { - open: "".to_owned()), - }, - ), - ( - "xsl".to_owned(), - LanguageComment { - open: "".to_owned()), - }, - ), - ( - "yaml".to_owned(), - LanguageComment { - open: "#".to_owned(), - close: None, - }, - ), - ]) -} diff --git a/src/main.rs b/src/main.rs index ae1c3af..9df1cc9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,3 @@ -mod language_comments; - -use language_comments::build_language_comments; use reqwest::header::AUTHORIZATION; use ropey::Rope; use serde::{Deserialize, Serialize}; @@ -18,12 +15,6 @@ use tracing::{error, info}; use tracing_appender::rolling; use tracing_subscriber::EnvFilter; -#[derive(Clone, Debug, Deserialize)] -pub struct LanguageComment { - open: String, - close: Option, -} - #[derive(Clone, Debug, Deserialize, Serialize)] struct RequestParams { max_new_tokens: u32, @@ -112,7 +103,6 @@ struct Backend { document_map: Arc>>, http_client: reqwest::Client, workspace_folders: Arc>>>, - language_comments: HashMap, tokenizer_map: Arc>>, } @@ -143,45 +133,10 @@ fn internal_error(err: E) -> Error { } } -fn file_path_comment( - file_url: Url, - file_language_id: &str, - workspace_folders: Option<&Vec>, - language_comments: &HashMap, -) -> String { - let mut file_path = file_url.path().to_owned(); - let path_in_workspace = if let Some(workspace_folders) = workspace_folders { - for workspace_folder in workspace_folders { - let workspace_folder_path = workspace_folder.uri.path(); - if file_path.starts_with(workspace_folder_path) { - file_path = file_path.replace(workspace_folder_path, ""); - break; - } - } - file_path - } else { - file_path - }; - let lc = match language_comments.get(file_language_id) { - Some(id) => id.clone(), - None => LanguageComment { - open: "//".to_owned(), - close: None, - }, - }; - let close = if let Some(close) = lc.close { - format!(" {close}") - } else { - "".to_owned() - }; - format!("{} {path_in_workspace}{close}\n", lc.open) -} - fn build_prompt( pos: Position, text: &Rope, fim: &FimParams, - file_path: String, tokenizer: Tokenizer, context_window: usize, ) -> Result { @@ -229,8 +184,7 @@ fn build_prompt( after_line = after_iter.next(); } Ok(format!( - "{}{}{}{}{}{}", - file_path, + "{}{}{}{}{}", fim.prefix, before.into_iter().rev().collect::>().join(""), fim.suffix, @@ -257,11 +211,7 @@ fn build_prompt( token_count -= tokens; before.push(line); } - Ok(format!( - "{}{}", - file_path, - &before.into_iter().rev().collect::>().join("") - )) + Ok(before.into_iter().rev().collect::>().join("")) } } @@ -382,12 +332,6 @@ impl Backend { let document = document_map .get(params.text_document_position.text_document.uri.as_str()) .ok_or_else(|| internal_error("failed to find document"))?; - let file_path = file_path_comment( - params.text_document_position.text_document.uri, - &document.language_id, - self.workspace_folders.read().await.as_ref(), - &self.language_comments, - ); let tokenizer = get_tokenizer( ¶ms.model, &mut *self.tokenizer_map.write().await, @@ -401,7 +345,6 @@ impl Backend { params.text_document_position.position, &document.text, ¶ms.fim, - file_path, tokenizer, params.context_window, )?; @@ -534,7 +477,6 @@ async fn main() { document_map: Arc::new(RwLock::new(HashMap::new())), http_client, workspace_folders: Arc::new(RwLock::new(None)), - language_comments: build_language_comments(), tokenizer_map: Arc::new(RwLock::new(HashMap::new())), }) .custom_method("llm-ls/getCompletions", Backend::get_completions)