gered/llm-ls
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refactor: remove file path header (#7)

Browse source
This commit is contained in:
Luc Georges 2023-08-27 20:13:35 +02:00 committed by GitHub
parent de459b3789
commit ba4b20e6d5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 3 additions and 474 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Cargo.lock generated
View file

@ -1021,7 +1021,7 @@ checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503"
[[package]] [[package]]
name = "llm-ls" name = "llm-ls"
version = "0.1.0" version = "0.1.1"
dependencies = [ dependencies = [
"home", "home",
"reqwest", "reqwest",

413
src/language_comments.rs
View file

@ -1,413 +0,0 @@
use crate::LanguageComment;
use std::collections::HashMap;
pub fn build_language_comments() -> HashMap<String, LanguageComment> {
HashMap::from([
(
"abap".to_owned(),
LanguageComment {
open: "*".to_owned(),
close: None,
},
),
(
"bat".to_owned(),
LanguageComment {
open: "REM".to_owned(),
close: None,
},
),
(
"bibtex".to_owned(),
LanguageComment {
open: "%".to_owned(),
close: None,
},
),
(
"clojure".to_owned(),
LanguageComment {
open: ";;".to_owned(),
close: None,
},
),
(
"coffeescript".to_owned(),
LanguageComment {
open: "#".to_owned(),
close: None,
},
),
(
"c".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"cpp".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"csharp".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"css".to_owned(),
LanguageComment {
open: "/*".to_owned(),
close: Some("*/".to_owned()),
},
),
(
"diff".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"dart".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"dockerfile".to_owned(),
LanguageComment {
open: "#".to_owned(),
close: None,
},
),
(
"elixir".to_owned(),
LanguageComment {
open: "#".to_owned(),
close: None,
},
),
(
"erlang".to_owned(),
LanguageComment {
open: "%".to_owned(),
close: None,
},
),
(
"fsharp".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"git-commit".to_owned(),
LanguageComment {
open: "#".to_owned(),
close: None,
},
),
(
"git-rebase".to_owned(),
LanguageComment {
open: "#".to_owned(),
close: None,
},
),
(
"go".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"groovy".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"handlebars".to_owned(),
LanguageComment {
open: "{{!--".to_owned(),
close: Some("--}}".to_owned()),
},
),
(
"html".to_owned(),
LanguageComment {
open: "<!--".to_owned(),
close: Some("-->".to_owned()),
},
),
(
"ini".to_owned(),
LanguageComment {
open: ";".to_owned(),
close: None,
},
),
(
"java".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"javascript".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"javascriptreact".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"json".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"latex".to_owned(),
LanguageComment {
open: "%".to_owned(),
close: None,
},
),
(
"less".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"lua".to_owned(),
LanguageComment {
open: "--".to_owned(),
close: None,
},
),
(
"makefile".to_owned(),
LanguageComment {
open: "#".to_owned(),
close: None,
},
),
(
"markdown".to_owned(),
LanguageComment {
open: "<!--".to_owned(),
close: Some("-->".to_owned()),
},
),
(
"objective-c".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"objective-cpp".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"perl".to_owned(),
LanguageComment {
open: "#".to_owned(),
close: None,
},
),
(
"perl6".to_owned(),
LanguageComment {
open: "#".to_owned(),
close: None,
},
),
(
"php".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"powershell".to_owned(),
LanguageComment {
open: "#".to_owned(),
close: None,
},
),
(
"jade".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"python".to_owned(),
LanguageComment {
open: "#".to_owned(),
close: None,
},
),
(
"r".to_owned(),
LanguageComment {
open: "#".to_owned(),
close: None,
},
),
(
"razor".to_owned(),
LanguageComment {
open: "@*".to_owned(),
close: Some("*@".to_owned()),
},
),
(
"ruby".to_owned(),
LanguageComment {
open: "#".to_owned(),
close: None,
},
),
(
"rust".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"scss".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"sass".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"scala".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"shaderlab".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"shellscript".to_owned(),
LanguageComment {
open: "#".to_owned(),
close: None,
},
),
(
"sql".to_owned(),
LanguageComment {
open: "--".to_owned(),
close: None,
},
),
(
"swift".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"toml".to_owned(),
LanguageComment {
open: "#".to_owned(),
close: None,
},
),
(
"typescript".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"typescriptreact".to_owned(),
LanguageComment {
open: "//".to_owned(),
close: None,
},
),
(
"tex".to_owned(),
LanguageComment {
open: "%".to_owned(),
close: None,
},
),
(
"vb".to_owned(),
LanguageComment {
open: "'".to_owned(),
close: None,
},
),
(
"xml".to_owned(),
LanguageComment {
open: "<!--".to_owned(),
close: Some("-->".to_owned()),
},
),
(
"xsl".to_owned(),
LanguageComment {
open: "<!--".to_owned(),
close: Some("-->".to_owned()),
},
),
(
"yaml".to_owned(),
LanguageComment {
open: "#".to_owned(),
close: None,
},
),
])
}

62
src/main.rs
View file

@ -1,6 +1,3 @@
mod language_comments;
use language_comments::build_language_comments;
use reqwest::header::AUTHORIZATION; use reqwest::header::AUTHORIZATION;
use ropey::Rope; use ropey::Rope;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -18,12 +15,6 @@ use tracing::{error, info};
use tracing_appender::rolling; use tracing_appender::rolling;
use tracing_subscriber::EnvFilter; use tracing_subscriber::EnvFilter;
#[derive(Clone, Debug, Deserialize)]
pub struct LanguageComment {
open: String,
close: Option<String>,
}
#[derive(Clone, Debug, Deserialize, Serialize)] #[derive(Clone, Debug, Deserialize, Serialize)]
struct RequestParams { struct RequestParams {
max_new_tokens: u32, max_new_tokens: u32,
@ -112,7 +103,6 @@ struct Backend {
document_map: Arc<RwLock<HashMap<String, Document>>>, document_map: Arc<RwLock<HashMap<String, Document>>>,
http_client: reqwest::Client, http_client: reqwest::Client,
workspace_folders: Arc<RwLock<Option<Vec<WorkspaceFolder>>>>, workspace_folders: Arc<RwLock<Option<Vec<WorkspaceFolder>>>>,
language_comments: HashMap<String, LanguageComment>,
tokenizer_map: Arc<RwLock<HashMap<String, Tokenizer>>>, tokenizer_map: Arc<RwLock<HashMap<String, Tokenizer>>>,
} }
@ -143,45 +133,10 @@ fn internal_error<E: Display>(err: E) -> Error {
} }
} }
fn file_path_comment(
file_url: Url,
file_language_id: &str,
workspace_folders: Option<&Vec<WorkspaceFolder>>,
language_comments: &HashMap<String, LanguageComment>,
) -> String {
let mut file_path = file_url.path().to_owned();
let path_in_workspace = if let Some(workspace_folders) = workspace_folders {
for workspace_folder in workspace_folders {
let workspace_folder_path = workspace_folder.uri.path();
if file_path.starts_with(workspace_folder_path) {
file_path = file_path.replace(workspace_folder_path, "");
break;
}
}
file_path
} else {
file_path
};
let lc = match language_comments.get(file_language_id) {
Some(id) => id.clone(),
None => LanguageComment {
open: "//".to_owned(),
close: None,
},
};
let close = if let Some(close) = lc.close {
format!(" {close}")
} else {
"".to_owned()
};
format!("{} {path_in_workspace}{close}\n", lc.open)
}
fn build_prompt( fn build_prompt(
pos: Position, pos: Position,
text: &Rope, text: &Rope,
fim: &FimParams, fim: &FimParams,
file_path: String,
tokenizer: Tokenizer, tokenizer: Tokenizer,
context_window: usize, context_window: usize,
) -> Result<String> { ) -> Result<String> {
@ -229,8 +184,7 @@ fn build_prompt(
after_line = after_iter.next(); after_line = after_iter.next();
} }
Ok(format!( Ok(format!(
"{}{}{}{}{}{}", "{}{}{}{}{}",
file_path,
fim.prefix, fim.prefix,
before.into_iter().rev().collect::<Vec<_>>().join(""), before.into_iter().rev().collect::<Vec<_>>().join(""),
fim.suffix, fim.suffix,
@ -257,11 +211,7 @@ fn build_prompt(
token_count -= tokens; token_count -= tokens;
before.push(line); before.push(line);
} }
Ok(format!( Ok(before.into_iter().rev().collect::<Vec<_>>().join(""))
"{}{}",
file_path,
&before.into_iter().rev().collect::<Vec<_>>().join("")
))
} }
} }
@ -382,12 +332,6 @@ impl Backend {
let document = document_map let document = document_map
.get(params.text_document_position.text_document.uri.as_str()) .get(params.text_document_position.text_document.uri.as_str())
.ok_or_else(|| internal_error("failed to find document"))?; .ok_or_else(|| internal_error("failed to find document"))?;
let file_path = file_path_comment(
params.text_document_position.text_document.uri,
&document.language_id,
self.workspace_folders.read().await.as_ref(),
&self.language_comments,
);
let tokenizer = get_tokenizer( let tokenizer = get_tokenizer(
&params.model, &params.model,
&mut *self.tokenizer_map.write().await, &mut *self.tokenizer_map.write().await,
@ -401,7 +345,6 @@ impl Backend {
params.text_document_position.position, params.text_document_position.position,
&document.text, &document.text,
&params.fim, &params.fim,
file_path,
tokenizer, tokenizer,
params.context_window, params.context_window,
)?; )?;
@ -534,7 +477,6 @@ async fn main() {
document_map: Arc::new(RwLock::new(HashMap::new())), document_map: Arc::new(RwLock::new(HashMap::new())),
http_client, http_client,
workspace_folders: Arc::new(RwLock::new(None)), workspace_folders: Arc::new(RwLock::new(None)),
language_comments: build_language_comments(),
tokenizer_map: Arc::new(RwLock::new(HashMap::new())), tokenizer_map: Arc::new(RwLock::new(HashMap::new())),
}) })
.custom_method("llm-ls/getCompletions", Backend::get_completions) .custom_method("llm-ls/getCompletions", Backend::get_completions)