test: fix invalid deserialization (#85)

* test: fix invalid deserialization * feat: install node or python only when needed * feat: when action is rerun, run testbed with logs in debug * feat(ci): update actions to node 20 * feat: log stdout & stderr from setup & build cmd as debug * fix: bump helix revision * fix: use helix fork for the time being * fix(ci): install missing setuptools * fix(ci): revert back to python 3.10 * fix(ci): disable lance
2024-02-19 17:25:42 +01:00 · 2024-02-19 17:25:42 +01:00 · 386ed53183
parent 0d035901f0
commit 386ed53183
5 changed files with 81 additions and 53 deletions
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@ -14,19 +14,19 @@ jobs:
    strategy:
      matrix:
        repo:
-          - { name: simple, key: simple, parallel: 8 }
-          - { name: mmaitre314/picklescan, key: picklescan, parallel: 8 }
-          - { name: huggingface/huggingface_hub, key: huggingface_hub, parallel: 8 }
-          - { name: tiangolo/fastapi, key: fastapi, parallel: 8 }
-          - { name: encode/starlette, key: starlette, parallel: 8 }
-          - { name: lancedb/lancedb, key: lancedb, parallel: 2 }
-          - { name: lancedb/lance, key: lance, parallel: 2 }
-          - { name: tkaitchuck/constrandom, key: constrandom, parallel: 8 }
-          - { name: jaemk/cached, key: cached, parallel: 4 }
-          - { name: smol-rs/async-executor, key: async-executor, parallel: 4 }
-          - { name: gcanti/io-ts, key: io-ts, parallel: 8 }
-          - { name: colinhacks/zod, key: zod, parallel: 8 }
-          - { name: helix-editor/helix, key: helix, parallel: 2 }
+          - { name: simple, key: simple, parallel: 8, node: false, python: false }
+          - { name: mmaitre314/picklescan, key: picklescan, parallel: 8, node: false, python: true }
+          - { name: huggingface/huggingface_hub, key: huggingface_hub, parallel: 8, node: false, python: true }
+          - { name: tiangolo/fastapi, key: fastapi, parallel: 8, node: false, python: true }
+          - { name: encode/starlette, key: starlette, parallel: 8, node: false, python: true }
+          - { name: lancedb/lancedb, key: lancedb, parallel: 2, node: false, python: false }
+          # - { name: lancedb/lance, key: lance, parallel: 2, node: false, python: false }
+          - { name: tkaitchuck/constrandom, key: constrandom, parallel: 8, node: false, python: false }
+          - { name: jaemk/cached, key: cached, parallel: 4, node: false, python: false }
+          - { name: smol-rs/async-executor, key: async-executor, parallel: 4, node: false, python: false }
+          - { name: gcanti/io-ts, key: io-ts, parallel: 8, node: true, python: false }
+          - { name: colinhacks/zod, key: zod, parallel: 8, node: true, python: false }
+          - { name: helix-editor/helix, key: helix, parallel: 2, node: false, python: false }
    runs-on: [self-hosted, intel-cpu, 8-cpu, ci]
    container:
      image: ubuntu:22.04
@ -46,21 +46,24 @@ jobs:
          toolchain: nightly

      - name: Install Python 3.10
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
+        if: matrix.repo.python
        with:
          python-version: '3.10'

      - name: Install node 18
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
+        if: matrix.repo.node
        with:
          node-version: 18

      - name: Install yarn
+        if: matrix.repo.node
        run: |
          npm i -g yarn

      - name: Set up cargo cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
        continue-on-error: false
        with:
          path: |
@ -80,15 +83,17 @@ jobs:
        if: github.event_name == 'push' || github.event_name == 'pull_request'
        env:
          API_TOKEN: ${{ secrets.API_TOKEN }}
+          LOG_LEVEL: ${{ github.run_attempt > 1 && 'debug' || 'info' }}

      - name: Run testbed
        run: 'cargo run --bin testbed -r -- --api-token $API_TOKEN -f ${{ matrix.repo.name }} -p ${{ matrix.repo.parallel }}'
        if: github.event_name == 'workflow_dispatch'
        env:
          API_TOKEN: ${{ secrets.API_TOKEN }}
+          LOG_LEVEL: ${{ github.run_attempt > 1 && 'debug' || 'info' }}

      - name: Upload artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
        with:
          name: results-${{ matrix.repo.key }}
          path: ./results.json
@ -104,67 +109,67 @@ jobs:
          apt update
          apt install -y jq

-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
        with:
          name: results-simple
          path: results-simple

-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
        with:
          name: results-picklescan
          path: results-picklescan

-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
        with:
          name: results-huggingface_hub
          path: results-huggingface_hub

-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
        with:
          name: results-fastapi
          path: results-fastapi

-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
        with:
          name: results-starlette
          path: results-starlette

-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
        with:
          name: results-lancedb
          path: results-lancedb

-      - uses: actions/download-artifact@v1
-        with:
-          name: results-lance
-          path: results-lance
+      # - uses: actions/download-artifact@v4
+      #   with:
+      #     name: results-lance
+      #     path: results-lance

-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
        with:
          name: results-constrandom
          path: results-constrandom

-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
        with:
          name: results-cached
          path: results-cached

-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
        with:
          name: results-async-executor
          path: results-async-executor

-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
        with:
          name: results-io-ts
          path: results-io-ts

-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
        with:
          name: results-zod
          path: results-zod

-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
        with:
          name: results-helix
          path: results-helix
--- a/crates/lsp-client/src/msg.rs
+++ b/crates/lsp-client/src/msg.rs
@ -1,7 +1,6 @@
 use std::{
    fmt::{self, Display},
    io,
-    marker::Unpin,
 };

 use serde::{de::DeserializeOwned, Deserialize, Serialize};
--- a/crates/testbed/repositories-ci.yaml
+++ b/crates/testbed/repositories-ci.yaml
@ -30,7 +30,7 @@ repositories:
      type: github
      owner: mmaitre314
      name: picklescan
-      revision: 40001cd1caa9e041b1bce1b80f3707056cd8be52
+      revision: 108b06040de644c7aeb5f22d8b10dd9c5d282386
      src_path: src/picklescan
    build_command: picklescan-venv/bin/python3
    build_args: ["-m", "compileall", "-q", "."]
@ -201,9 +201,9 @@ repositories:
    holes_file: zod-smol.json
  - source:
      type: github
-      owner: helix-editor
+      owner: McPatate
      name: helix
-      revision: a1272bdb17a63361342a318982e46129d558743c
+      revision: 01c1ebebd813c9034059a966fae2bbc487b1a69f
      exclude_paths:
        - .cargo
        - .github
--- a/crates/testbed/repositories.yaml
+++ b/crates/testbed/repositories.yaml
@ -30,7 +30,7 @@ repositories:
      type: github
      owner: mmaitre314
      name: picklescan
-      revision: 40001cd1caa9e041b1bce1b80f3707056cd8be52
+      revision: 108b06040de644c7aeb5f22d8b10dd9c5d282386
      src_path: src/picklescan
    build_command: picklescan-venv/bin/python3
    build_args: ["-m", "compileall", "-q", "."]
@ -201,9 +201,9 @@ repositories:
    holes_file: zod.json
  - source:
      type: github
-      owner: helix-editor
+      owner: McPatate
      name: helix
-      revision: a1272bdb17a63361342a318982e46129d558743c
+      revision: 01c1ebebd813c9034059a966fae2bbc487b1a69f
      exclude_paths:
        - .cargo
        - .github
--- a/crates/testbed/src/main.rs
+++ b/crates/testbed/src/main.rs
@ -28,7 +28,11 @@ use serde_json::{Map, Value};
 use tempfile::TempDir;
 use tokio::{
    fs::{self, read_to_string, File, OpenOptions},
-    io::{self, AsyncReadExt, AsyncWriteExt},
+    io::{
+        self, AsyncBufRead, AsyncBufReadExt, AsyncReadExt, AsyncWriteExt,
+        BufReader as TokioBufReader,
+    },
+    join,
    process::Command,
    sync::{OnceCell, RwLock, Semaphore},
 };
@ -198,6 +202,7 @@ struct RepositoriesConfig {
    context_window: usize,
    fim: FimParams,
    model: String,
+    #[serde(flatten)]
    backend: Backend,
    repositories: Vec<Repository>,
    tls_skip_verify_insecure: bool,
@ -400,15 +405,20 @@ async fn run_setup(
            command.0,
            command.1.join(" ")
        );
-        let status = status_cmd
+        let mut child = status_cmd
            .args(&command.1)
            .current_dir(&repo_path)
-            .stdout(Stdio::null())
-            .stderr(Stdio::null())
-            .spawn()?
-            .wait()
-            .await?;
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?;

+        if let (Some(stdout), Some(stderr)) = (child.stdout.take(), child.stderr.take()) {
+            let stdout = TokioBufReader::new(stdout);
+            let stderr = TokioBufReader::new(stderr);
+            join!(log_lines(stdout), log_lines(stderr));
+        }
+
+        let status = child.wait().await?;
        if !status.success() {
            return Err(anyhow!(
                "error running: \"{} {}\"",
@ -432,17 +442,31 @@ async fn build(
        status_cmd.env(name, value);
    }
    debug!("building repo: {command} {args:?}");
-    let status = status_cmd
+
+    let mut child = status_cmd
        .args(args)
        .current_dir(repo_path)
-        .stdout(Stdio::null())
-        .stderr(Stdio::null())
-        .spawn()?
-        .wait()
-        .await?;
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()?;
+
+    if let (Some(stdout), Some(stderr)) = (child.stdout.take(), child.stderr.take()) {
+        let stdout = TokioBufReader::new(stdout);
+        let stderr = TokioBufReader::new(stderr);
+        join!(log_lines(stdout), log_lines(stderr));
+    }
+
+    let status = child.wait().await?;
    Ok(status.success())
 }

+async fn log_lines<R: AsyncReadExt + AsyncBufRead + Unpin>(stdio: R) {
+    let mut lines = stdio.lines();
+    while let Ok(Some(log)) = lines.next_line().await {
+        debug!("{log}");
+    }
+}
+
 #[allow(clippy::too_many_arguments)]
 async fn complete_holes(
    hole: Hole,