gered/llm-ls
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

test: fix invalid deserialization (#85)

Browse source 
* test: fix invalid deserialization

* feat: install node or python only when needed

* feat: when action is rerun, run testbed with logs in debug

* feat(ci): update actions to node 20

* feat: log stdout & stderr from setup & build cmd as debug

* fix: bump helix revision

* fix: use helix fork for the time being

* fix(ci): install missing setuptools

* fix(ci): revert back to python 3.10

* fix(ci): disable lance
This commit is contained in:
Luc Georges 2024-02-19 17:25:42 +01:00 committed by GitHub
parent 0d035901f0
commit 386ed53183
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 81 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

71
.github/workflows/test.yml vendored
View file

@ -14,19 +14,19 @@ jobs:
strategy: strategy:
matrix: matrix:
repo: repo:
- { name: simple, key: simple, parallel: 8 } - { name: simple, key: simple, parallel: 8, node: false, python: false }
- { name: mmaitre314/picklescan, key: picklescan, parallel: 8 } - { name: mmaitre314/picklescan, key: picklescan, parallel: 8, node: false, python: true }
- { name: huggingface/huggingface_hub, key: huggingface_hub, parallel: 8 } - { name: huggingface/huggingface_hub, key: huggingface_hub, parallel: 8, node: false, python: true }
- { name: tiangolo/fastapi, key: fastapi, parallel: 8 } - { name: tiangolo/fastapi, key: fastapi, parallel: 8, node: false, python: true }
- { name: encode/starlette, key: starlette, parallel: 8 } - { name: encode/starlette, key: starlette, parallel: 8, node: false, python: true }
- { name: lancedb/lancedb, key: lancedb, parallel: 2 } - { name: lancedb/lancedb, key: lancedb, parallel: 2, node: false, python: false }
- { name: lancedb/lance, key: lance, parallel: 2 } # - { name: lancedb/lance, key: lance, parallel: 2, node: false, python: false }
- { name: tkaitchuck/constrandom, key: constrandom, parallel: 8 } - { name: tkaitchuck/constrandom, key: constrandom, parallel: 8, node: false, python: false }
- { name: jaemk/cached, key: cached, parallel: 4 } - { name: jaemk/cached, key: cached, parallel: 4, node: false, python: false }
- { name: smol-rs/async-executor, key: async-executor, parallel: 4 } - { name: smol-rs/async-executor, key: async-executor, parallel: 4, node: false, python: false }
- { name: gcanti/io-ts, key: io-ts, parallel: 8 } - { name: gcanti/io-ts, key: io-ts, parallel: 8, node: true, python: false }
- { name: colinhacks/zod, key: zod, parallel: 8 } - { name: colinhacks/zod, key: zod, parallel: 8, node: true, python: false }
- { name: helix-editor/helix, key: helix, parallel: 2 } - { name: helix-editor/helix, key: helix, parallel: 2, node: false, python: false }
runs-on: [self-hosted, intel-cpu, 8-cpu, ci] runs-on: [self-hosted, intel-cpu, 8-cpu, ci]
container: container:
image: ubuntu:22.04 image: ubuntu:22.04
@ -46,21 +46,24 @@ jobs:
toolchain: nightly toolchain: nightly
- name: Install Python 3.10 - name: Install Python 3.10
uses: actions/setup-python@v4 uses: actions/setup-python@v5
if: matrix.repo.python
with: with:
python-version: '3.10' python-version: '3.10'
- name: Install node 18 - name: Install node 18
uses: actions/setup-node@v3 uses: actions/setup-node@v4
if: matrix.repo.node
with: with:
node-version: 18 node-version: 18
- name: Install yarn - name: Install yarn
if: matrix.repo.node
run: | run: |
npm i -g yarn npm i -g yarn
- name: Set up cargo cache - name: Set up cargo cache
uses: actions/cache@v3 uses: actions/cache@v4
continue-on-error: false continue-on-error: false
with: with:
path: | path: |
@ -80,15 +83,17 @@ jobs:
if: github.event_name == 'push' || github.event_name == 'pull_request' if: github.event_name == 'push' || github.event_name == 'pull_request'
env: env:
API_TOKEN: ${{ secrets.API_TOKEN }} API_TOKEN: ${{ secrets.API_TOKEN }}
LOG_LEVEL: ${{ github.run_attempt > 1 && 'debug' || 'info' }}
- name: Run testbed - name: Run testbed
run: 'cargo run --bin testbed -r -- --api-token $API_TOKEN -f ${{ matrix.repo.name }} -p ${{ matrix.repo.parallel }}' run: 'cargo run --bin testbed -r -- --api-token $API_TOKEN -f ${{ matrix.repo.name }} -p ${{ matrix.repo.parallel }}'
if: github.event_name == 'workflow_dispatch' if: github.event_name == 'workflow_dispatch'
env: env:
API_TOKEN: ${{ secrets.API_TOKEN }} API_TOKEN: ${{ secrets.API_TOKEN }}
LOG_LEVEL: ${{ github.run_attempt > 1 && 'debug' || 'info' }}
- name: Upload artifacts - name: Upload artifacts
uses: actions/upload-artifact@v1 uses: actions/upload-artifact@v4
with: with:
name: results-${{ matrix.repo.key }} name: results-${{ matrix.repo.key }}
path: ./results.json path: ./results.json
@ -104,67 +109,67 @@ jobs:
apt update apt update
apt install -y jq apt install -y jq
- uses: actions/download-artifact@v1 - uses: actions/download-artifact@v4
with: with:
name: results-simple name: results-simple
path: results-simple path: results-simple
- uses: actions/download-artifact@v1 - uses: actions/download-artifact@v4
with: with:
name: results-picklescan name: results-picklescan
path: results-picklescan path: results-picklescan
- uses: actions/download-artifact@v1 - uses: actions/download-artifact@v4
with: with:
name: results-huggingface_hub name: results-huggingface_hub
path: results-huggingface_hub path: results-huggingface_hub
- uses: actions/download-artifact@v1 - uses: actions/download-artifact@v4
with: with:
name: results-fastapi name: results-fastapi
path: results-fastapi path: results-fastapi
- uses: actions/download-artifact@v1 - uses: actions/download-artifact@v4
with: with:
name: results-starlette name: results-starlette
path: results-starlette path: results-starlette
- uses: actions/download-artifact@v1 - uses: actions/download-artifact@v4
with: with:
name: results-lancedb name: results-lancedb
path: results-lancedb path: results-lancedb
- uses: actions/download-artifact@v1 # - uses: actions/download-artifact@v4
with: # with:
name: results-lance # name: results-lance
path: results-lance # path: results-lance
- uses: actions/download-artifact@v1 - uses: actions/download-artifact@v4
with: with:
name: results-constrandom name: results-constrandom
path: results-constrandom path: results-constrandom
- uses: actions/download-artifact@v1 - uses: actions/download-artifact@v4
with: with:
name: results-cached name: results-cached
path: results-cached path: results-cached
- uses: actions/download-artifact@v1 - uses: actions/download-artifact@v4
with: with:
name: results-async-executor name: results-async-executor
path: results-async-executor path: results-async-executor
- uses: actions/download-artifact@v1 - uses: actions/download-artifact@v4
with: with:
name: results-io-ts name: results-io-ts
path: results-io-ts path: results-io-ts
- uses: actions/download-artifact@v1 - uses: actions/download-artifact@v4
with: with:
name: results-zod name: results-zod
path: results-zod path: results-zod
- uses: actions/download-artifact@v1 - uses: actions/download-artifact@v4
with: with:
name: results-helix name: results-helix
path: results-helix path: results-helix

1
crates/lsp-client/src/msg.rs
View file

@ -1,7 +1,6 @@
use std::{ use std::{
fmt::{self, Display}, fmt::{self, Display},
io, io,
marker::Unpin,
}; };
use serde::{de::DeserializeOwned, Deserialize, Serialize}; use serde::{de::DeserializeOwned, Deserialize, Serialize};

6
crates/testbed/repositories-ci.yaml
View file

@ -30,7 +30,7 @@ repositories:
type: github type: github
owner: mmaitre314 owner: mmaitre314
name: picklescan name: picklescan
revision: 40001cd1caa9e041b1bce1b80f3707056cd8be52 revision: 108b06040de644c7aeb5f22d8b10dd9c5d282386
src_path: src/picklescan src_path: src/picklescan
build_command: picklescan-venv/bin/python3 build_command: picklescan-venv/bin/python3
build_args: ["-m", "compileall", "-q", "."] build_args: ["-m", "compileall", "-q", "."]
@ -201,9 +201,9 @@ repositories:
holes_file: zod-smol.json holes_file: zod-smol.json
- source: - source:
type: github type: github
owner: helix-editor owner: McPatate
name: helix name: helix
revision: a1272bdb17a63361342a318982e46129d558743c revision: 01c1ebebd813c9034059a966fae2bbc487b1a69f
exclude_paths: exclude_paths:
- .cargo - .cargo
- .github - .github

6
crates/testbed/repositories.yaml
View file

@ -30,7 +30,7 @@ repositories:
type: github type: github
owner: mmaitre314 owner: mmaitre314
name: picklescan name: picklescan
revision: 40001cd1caa9e041b1bce1b80f3707056cd8be52 revision: 108b06040de644c7aeb5f22d8b10dd9c5d282386
src_path: src/picklescan src_path: src/picklescan
build_command: picklescan-venv/bin/python3 build_command: picklescan-venv/bin/python3
build_args: ["-m", "compileall", "-q", "."] build_args: ["-m", "compileall", "-q", "."]
@ -201,9 +201,9 @@ repositories:
holes_file: zod.json holes_file: zod.json
- source: - source:
type: github type: github
owner: helix-editor owner: McPatate
name: helix name: helix
revision: a1272bdb17a63361342a318982e46129d558743c revision: 01c1ebebd813c9034059a966fae2bbc487b1a69f
exclude_paths: exclude_paths:
- .cargo - .cargo
- .github - .github

50
crates/testbed/src/main.rs
View file

@ -28,7 +28,11 @@ use serde_json::{Map, Value};
use tempfile::TempDir; use tempfile::TempDir;
use tokio::{ use tokio::{
fs::{self, read_to_string, File, OpenOptions}, fs::{self, read_to_string, File, OpenOptions},
io::{self, AsyncReadExt, AsyncWriteExt}, io::{
self, AsyncBufRead, AsyncBufReadExt, AsyncReadExt, AsyncWriteExt,
BufReader as TokioBufReader,
},
join,
process::Command, process::Command,
sync::{OnceCell, RwLock, Semaphore}, sync::{OnceCell, RwLock, Semaphore},
}; };
@ -198,6 +202,7 @@ struct RepositoriesConfig {
context_window: usize, context_window: usize,
fim: FimParams, fim: FimParams,
model: String, model: String,
#[serde(flatten)]
backend: Backend, backend: Backend,
repositories: Vec<Repository>, repositories: Vec<Repository>,
tls_skip_verify_insecure: bool, tls_skip_verify_insecure: bool,
@ -400,15 +405,20 @@ async fn run_setup(
command.0, command.0,
command.1.join(" ") command.1.join(" ")
); );
let status = status_cmd let mut child = status_cmd
.args(&command.1) .args(&command.1)
.current_dir(&repo_path) .current_dir(&repo_path)
.stdout(Stdio::null()) .stdout(Stdio::piped())
.stderr(Stdio::null()) .stderr(Stdio::piped())
.spawn()? .spawn()?;
.wait()
.await?;
if let (Some(stdout), Some(stderr)) = (child.stdout.take(), child.stderr.take()) {
let stdout = TokioBufReader::new(stdout);
let stderr = TokioBufReader::new(stderr);
join!(log_lines(stdout), log_lines(stderr));
}
let status = child.wait().await?;
if !status.success() { if !status.success() {
return Err(anyhow!( return Err(anyhow!(
"error running: \"{} {}\"", "error running: \"{} {}\"",
@ -432,17 +442,31 @@ async fn build(
status_cmd.env(name, value); status_cmd.env(name, value);
} }
debug!("building repo: {command} {args:?}"); debug!("building repo: {command} {args:?}");
let status = status_cmd
let mut child = status_cmd
.args(args) .args(args)
.current_dir(repo_path) .current_dir(repo_path)
.stdout(Stdio::null()) .stdout(Stdio::piped())
.stderr(Stdio::null()) .stderr(Stdio::piped())
.spawn()? .spawn()?;
.wait()
.await?; if let (Some(stdout), Some(stderr)) = (child.stdout.take(), child.stderr.take()) {
let stdout = TokioBufReader::new(stdout);
let stderr = TokioBufReader::new(stderr);
join!(log_lines(stdout), log_lines(stderr));
}
let status = child.wait().await?;
Ok(status.success()) Ok(status.success())
} }
async fn log_lines<R: AsyncReadExt + AsyncBufRead + Unpin>(stdio: R) {
let mut lines = stdio.lines();
while let Ok(Some(log)) = lines.next_line().await {
debug!("{log}");
}
}
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
async fn complete_holes( async fn complete_holes(
hole: Hole, hole: Hole,