feat: add CI for release (#8)

* feat: add release CI

* debug: disable `aarch64-unknown-linux-gnu` & add missing `pkgconfig`

* fix: add `g++` & `pkg-config`

* fix: install libssl-dev on ubuntu

* fix: add sudo in specific case

* fix: remove ssl dependency

* feat: update Cargo.lock

* fix: use gcc-multilib

* fix: disable aarch64-linux for now

* fix: disable problematic platforms

* fix: comment out `aarch64-pc-windows-msvc`

* fix: set allow deadcode for `Document.language_id`

* fix: set valid release tag

.cargo/config.toml

@@ -0,0 +1,2 @@
+[alias]
+xtask = "run --package xtask --bin xtask --"

.github/actions/github-release/Dockerfile

@@ -0,0 +1,8 @@
+FROM node:slim
+
+COPY . /action
+WORKDIR /action
+
+RUN npm install --production
+
+ENTRYPOINT ["node", "/action/main.js"]

.github/actions/github-release/README.md

@@ -0,0 +1,21 @@
+# github-release
+
+Copy-pasted from
+https://github.com/rust-lang/rust-analyzer/tree/2df30e1e07eafc1de0359566423f471920693a34/.github/actions/github-release
+
+An action used to publish GitHub releases for `wasmtime`.
+
+As of the time of this writing there's a few actions floating around which
+perform github releases but they all tend to have their set of drawbacks.
+Additionally nothing handles deleting releases which we need for our rolling
+`dev` release.
+
+To handle all this, this action rolls its own implementation using the
+actions/toolkit repository and packages published there. These run in a Docker
+container and take various inputs to orchestrate the release from the build.
+
+More comments can be found in `main.js`.
+
+Testing this is really hard. If you want to try though run `npm install` and
+then `node main.js`. You'll have to configure a bunch of env vars though to get
+anything reasonably working.

.github/actions/github-release/action.yml

@@ -0,0 +1,15 @@
+name: 'wasmtime github releases'
+description: 'wasmtime github releases'
+inputs:
+  token:
+    description: ''
+    required: true
+  name:
+    description: ''
+    required: true
+  files:
+    description: ''
+    required: true
+runs:
+  using: 'docker'
+  image: 'Dockerfile'

.github/actions/github-release/main.js

@@ -0,0 +1,144 @@
+const core = require('@actions/core');
+const path = require("path");
+const fs = require("fs");
+const github = require('@actions/github');
+const glob = require('glob');
+
+function sleep(milliseconds) {
+  return new Promise(resolve => setTimeout(resolve, milliseconds));
+}
+
+async function runOnce() {
+  // Load all our inputs and env vars. Note that `getInput` reads from `INPUT_*`
+  const files = core.getInput('files');
+  const name = core.getInput('name');
+  const token = core.getInput('token');
+  const slug = process.env.GITHUB_REPOSITORY;
+  const owner = slug.split('/')[0];
+  const repo = slug.split('/')[1];
+  const sha = process.env.HEAD_SHA;
+
+  core.info(`files: ${files}`);
+  core.info(`name: ${name}`);
+
+  const options = {
+    request: {
+      timeout: 30000,
+    }
+  };
+  const octokit = github.getOctokit(token, options);
+
+  // Delete the previous release since we can't overwrite one. This may happen
+  // due to retrying an upload or it may happen because we're doing the dev
+  // release.
+  const releases = await octokit.paginate("GET /repos/:owner/:repo/releases", { owner, repo });
+  for (const release of releases) {
+    if (release.tag_name !== name) {
+      continue;
+    }
+    const release_id = release.id;
+    core.info(`deleting release ${release_id}`);
+    await octokit.rest.repos.deleteRelease({ owner, repo, release_id });
+  }
+
+  // We also need to update the `dev` tag while we're at it on the `dev` branch.
+  if (name == 'nightly') {
+    try {
+      core.info(`updating nightly tag`);
+      await octokit.rest.git.updateRef({
+        owner,
+        repo,
+        ref: 'tags/nightly',
+        sha,
+        force: true,
+      });
+    } catch (e) {
+      core.error(e);
+      core.info(`creating nightly tag`);
+      await octokit.rest.git.createTag({
+        owner,
+        repo,
+        tag: 'nightly',
+        message: 'nightly release',
+        object: sha,
+        type: 'commit',
+      });
+    }
+  }
+
+  // Creates an official GitHub release for this `tag`, and if this is `dev`
+  // then we know that from the previous block this should be a fresh release.
+  core.info(`creating a release`);
+  const release = await octokit.rest.repos.createRelease({
+    owner,
+    repo,
+    name,
+    tag_name: name,
+    target_commitish: sha,
+    prerelease: name === 'nightly',
+  });
+  const release_id = release.data.id;
+
+  // Upload all the relevant assets for this release as just general blobs.
+  for (const file of glob.sync(files)) {
+    const size = fs.statSync(file).size;
+    const name = path.basename(file);
+
+    await runWithRetry(async function() {
+      // We can't overwrite assets, so remove existing ones from a previous try.
+      let assets = await octokit.rest.repos.listReleaseAssets({
+        owner,
+        repo,
+        release_id
+      });
+      for (const asset of assets.data) {
+        if (asset.name === name) {
+          core.info(`delete asset ${name}`);
+          const asset_id = asset.id;
+          await octokit.rest.repos.deleteReleaseAsset({ owner, repo, asset_id });
+        }
+      }
+
+      core.info(`upload ${file}`);
+      const headers = { 'content-length': size, 'content-type': 'application/octet-stream' };
+      const data = fs.createReadStream(file);
+      await octokit.rest.repos.uploadReleaseAsset({
+        data,
+        headers,
+        name,
+        url: release.data.upload_url,
+      });
+    });
+  }
+}
+
+async function runWithRetry(f) {
+  const retries = 10;
+  const maxDelay = 4000;
+  let delay = 1000;
+
+  for (let i = 0; i < retries; i++) {
+    try {
+      await f();
+      break;
+    } catch (e) {
+      if (i === retries - 1)
+        throw e;
+
+      core.error(e);
+      const currentDelay = Math.round(Math.random() * delay);
+      core.info(`sleeping ${currentDelay} ms`);
+      await sleep(currentDelay);
+      delay = Math.min(delay * 2, maxDelay);
+    }
+  }
+}
+
+async function run() {
+  await runWithRetry(runOnce);
+}
+
+run().catch(err => {
+  core.error(err);
+  core.setFailed(err.message);
+});

.github/actions/github-release/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "wasmtime-github-release",
+  "version": "0.0.0",
+  "main": "main.js",
+  "dependencies": {
+    "@actions/core": "^1.6",
+    "@actions/github": "^5.0",
+    "glob": "^7.1.5"
+  }
+}

.github/workflows/release.yml

@@ -0,0 +1,190 @@
+name: release
+on:
+  workflow_dispatch:
+
+  push:
+    branches:
+      - 'release/**'
+
+env:
+  CARGO_INCREMENTAL: 0
+  CARGO_NET_RETRY: 10
+  RUSTFLAGS: "-D warnings -W unreachable-pub"
+  RUSTUP_MAX_RETRIES: 10
+  FETCH_DEPTH: 0 # pull in the tags for the version string
+  MACOSX_DEPLOYMENT_TARGET: 10.15
+  CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
+  CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_LINKER: arm-linux-gnueabihf-gcc
+
+jobs:
+  dist:
+    strategy:
+      matrix:
+        include:
+          - os: windows-latest
+            target: x86_64-pc-windows-msvc
+            code-target: win32-x64
+          - os: windows-latest
+            target: i686-pc-windows-msvc
+            code-target: win32-ia32
+          # - os: windows-latest
+          #   target: aarch64-pc-windows-msvc
+          #   code-target: win32-arm64
+          # - os: ubuntu-20.04
+          #   target: x86_64-unknown-linux-gnu
+          #   code-target: linux-x64
+          #   container: ubuntu:18.04
+          # - os: ubuntu-20.04
+          #   target: aarch64-unknown-linux-gnu
+          #   code-target: linux-arm64
+          # - os: ubuntu-20.04
+          #   target: arm-unknown-linux-gnueabihf
+          #   code-target: linux-armhf
+          - os: macos-11
+            target: x86_64-apple-darwin
+            code-target: darwin-x64
+          - os: macos-11
+            target: aarch64-apple-darwin
+            code-target: darwin-arm64
+
+    env:
+      LLM_LS_TARGET: ${{ matrix.target }}
+
+    name: dist (${{ matrix.target }})
+    runs-on: ${{ matrix.os }}
+    container: ${{ matrix.container }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: ${{ env.FETCH_DEPTH }}
+
+      # - name: Install toolchain dependencies
+      #   if: matrix.container == 'ubuntu:18.04'
+      #   shell: bash
+      #   run: |
+      #     apt-get update && apt-get install -y build-essential curl
+      #     curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused -fsSL "https://sh.rustup.rs" | sh -s -- --profile minimal --default-toolchain none -y
+      #     echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH
+
+      - name: Install Rust toolchain
+        run: |
+          rustup update --no-self-update stable
+          rustup target add ${{ matrix.target }}
+          rustup component add rust-src
+      #
+      # - name: Update apt repositories
+      #   if: matrix.target == 'aarch64-unknown-linux-gnu' || matrix.target == 'arm-unknown-linux-gnueabihf'
+      #   run: sudo apt-get update
+
+      # - name: Install AArch64 target toolchain
+      #   if: matrix.target == 'aarch64-unknown-linux-gnu'
+      #   run: sudo apt-get install gcc-aarch64-linux-gnu
+
+      # - name: Install ARM target toolchain
+      #   if: matrix.target == 'arm-unknown-linux-gnueabihf'
+      #   run: sudo apt-get install gcc-multilib-arm-linux-gnueabihf
+
+      - name: Dist
+        run: cargo xtask dist
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v1
+        with:
+          name: dist-${{ matrix.target }}
+          path: ./dist
+
+  dist-x86_64-unknown-linux-musl:
+    name: dist (x86_64-unknown-linux-musl)
+    runs-on: ubuntu-latest
+    env:
+      LLM_LS_TARGET: x86_64-unknown-linux-musl
+      # For some reason `-crt-static` is not working for clang without lld
+      RUSTFLAGS: "-C link-arg=-fuse-ld=lld -C target-feature=-crt-static"
+    container:
+      image: rust:alpine
+      volumes:
+        - /usr/local/cargo/registry:/usr/local/cargo/registry
+
+    steps:
+      - name: Install dependencies
+        run: apk add --no-cache git clang lld musl-dev nodejs npm openssl-dev pkgconfig g++
+
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: ${{ env.FETCH_DEPTH }}
+
+      - name: Dist
+        run: cargo xtask dist
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v1
+        with:
+          name: dist-x86_64-unknown-linux-musl
+          path: ./dist
+
+  publish:
+    name: publish
+    runs-on: ubuntu-latest
+    needs: ["dist", "dist-x86_64-unknown-linux-musl"]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: ${{ env.FETCH_DEPTH }}
+
+      - run: echo "HEAD_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV
+      - run: 'echo "HEAD_SHA: $HEAD_SHA"'
+
+      - name: Split branch name
+        env:
+          BRANCH: ${{ github.ref_name }}
+        id: split
+        run: echo "::set-output name=tag::${BRANCH##*/}"
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: dist-aarch64-apple-darwin
+          path: dist
+      - uses: actions/download-artifact@v1
+        with:
+          name: dist-x86_64-apple-darwin
+          path: dist
+      # - uses: actions/download-artifact@v1
+      #   with:
+      #     name: dist-x86_64-unknown-linux-gnu
+      #     path: dist
+      - uses: actions/download-artifact@v1
+        with:
+          name: dist-x86_64-unknown-linux-musl
+          path: dist
+      # - uses: actions/download-artifact@v1
+      #   with:
+      #     name: dist-aarch64-unknown-linux-gnu
+      #     path: dist
+      # - uses: actions/download-artifact@v1
+      #   with:
+      #     name: dist-arm-unknown-linux-gnueabihf
+      #     path: dist
+      - uses: actions/download-artifact@v1
+        with:
+          name: dist-x86_64-pc-windows-msvc
+          path: dist
+      - uses: actions/download-artifact@v1
+        with:
+          name: dist-i686-pc-windows-msvc
+          path: dist
+      # - uses: actions/download-artifact@v1
+      #   with:
+      #     name: dist-aarch64-pc-windows-msvc
+      #     path: dist
+      - run: ls -al ./dist
+
+      - name: Publish Release
+        uses: ./.github/actions/github-release
+        with:
+          files: "dist/*"
+          name: ${{ steps.split.outputs.tag }}
+          token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -1 +1,2 @@
-/target
+dist/
+target/

Cargo.toml

@@ -1,19 +1,25 @@
-[package]
+[workspace]
-name = "llm-ls"
+members = ["xtask/", "crates/*"]
-version = "0.1.1"
+resolver = "2"
+
+[workspace.package]
+rust-version = "1.71"
 edition = "2021"
+license = "Apache-2.0"
+authors = ["Luc Georges <luc@huggingface.co>"]
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[profile.dev]
+# Disabling debug info speeds up builds a bunch,
+# and we don't rely on it for debugging that much.
+debug = 0
 
-[dependencies]
+[profile.dev.package]
-home = "0.5"
+# This speeds up `cargo xtask dist`.
-ropey = "1.6"
+miniz_oxide.opt-level = 3
-serde = { version = "1", features = ["derive"] }
-reqwest = { version = "0.11", features = ["json"] }
-tokenizers = "0.13"
-tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "rt-multi-thread"] }
-tower-lsp = "0.20"
-tracing = "0.1"
-tracing-appender = "0.2"
-tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
 
+[profile.release]
+incremental = true
+# Set this to 1 or 2 to get more useful backtraces in debugger.
+debug = 0
+
+[workspace.dependencies]

crates/llm-ls/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "llm-ls"
+version = "0.0.0"
+edition = "2021"
+
+[[bin]]
+name = "llm-ls"
+
+[dependencies]
+home = "0.5"
+ropey = "1.6"
+serde = { version = "1", features = ["derive"] }
+reqwest = { version = "0.11", default-features = false, features = ["json", "rustls-tls"] }
+tokenizers = { version = "0.13", default-features = false, features = ["onig"] }
+tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "rt-multi-thread"] }
+tower-lsp = "0.20"
+tracing = "0.1"
+tracing-appender = "0.2"
+tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
+

crates/llm-ls/src/main.rs

@@ -15,6 +15,9 @@ use tracing::{error, info};
 use tracing_appender::rolling;
 use tracing_subscriber::EnvFilter;
 
+// TODO:
+// * handle slice panic
+
 #[derive(Clone, Debug, Deserialize, Serialize)]
 struct RequestParams {
     max_new_tokens: u32,
@@ -86,6 +89,7 @@ enum APIResponse {
 
 #[derive(Debug)]
 struct Document {
+    #[allow(dead_code)]
     language_id: String,
     text: Rope,
 }

xtask/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "xtask"
+version = "0.1.0"
+publish = false
+rust-version.workspace = true
+edition.workspace = true
+license.workspace = true
+authors.workspace = true
+
+[dependencies]
+anyhow = "1"
+flate2 = "1"
+write-json = "0.1"
+xshell = "0.2"
+xflags = "0.3"
+time = { version = "0.3", default-features = false }
+zip = { version = "0.6", default-features = false, features = ["deflate", "time"] }

xtask/src/dist.rs

@@ -0,0 +1,147 @@
+use std::{
+    env,
+    fs::File,
+    io::{self, BufWriter},
+    path::{Path, PathBuf},
+};
+
+use flate2::{write::GzEncoder, Compression};
+use time::OffsetDateTime;
+use xshell::{cmd, Shell};
+use zip::{write::FileOptions, DateTime, ZipWriter};
+
+use crate::{flags, project_root};
+
+impl flags::Dist {
+    pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> {
+        let branch = sh.var("GITHUB_REF").unwrap_or_default();
+        let release = if branch.starts_with("refs/heads/release/") {
+            branch.replace("refs/heads/release/", "")
+        } else {
+            "0.0.0".to_owned()
+        };
+        let project_root = project_root();
+        let target = Target::get(&project_root);
+        let dist = project_root.join("dist");
+        sh.remove_path(&dist)?;
+        sh.create_dir(&dist)?;
+
+        dist_server(sh, &release, &target)?;
+        Ok(())
+    }
+}
+
+fn dist_server(sh: &Shell, release: &str, target: &Target) -> anyhow::Result<()> {
+    let _e = sh.push_env("CFG_RELEASE", release);
+    let _e = sh.push_env("CARGO_PROFILE_RELEASE_LTO", "thin");
+
+    // Uncomment to enable debug info for releases. Note that:
+    //   * debug info is split on windows and macs, so it does nothing for those platforms,
+    //   * on Linux, this blows up the binary size from 8MB to 43MB, which is unreasonable.
+    // let _e = sh.push_env("CARGO_PROFILE_RELEASE_DEBUG", "1");
+
+    if target.name.contains("-linux-") {
+        env::set_var("CC", "clang");
+    }
+
+    let target_name = &target.name;
+    cmd!(sh, "cargo build --manifest-path ./crates/llm-ls/Cargo.toml --bin llm-ls --target {target_name} --release").run()?;
+
+    let dst = Path::new("dist").join(&target.artifact_name);
+    gzip(&target.server_path, &dst.with_extension("gz"))?;
+    if target_name.contains("-windows-") {
+        zip(
+            &target.server_path,
+            target.symbols_path.as_ref(),
+            &dst.with_extension("zip"),
+        )?;
+    }
+
+    Ok(())
+}
+
+fn gzip(src_path: &Path, dest_path: &Path) -> anyhow::Result<()> {
+    let mut encoder = GzEncoder::new(File::create(dest_path)?, Compression::best());
+    let mut input = io::BufReader::new(File::open(src_path)?);
+    io::copy(&mut input, &mut encoder)?;
+    encoder.finish()?;
+    Ok(())
+}
+
+fn zip(src_path: &Path, symbols_path: Option<&PathBuf>, dest_path: &Path) -> anyhow::Result<()> {
+    let file = File::create(dest_path)?;
+    let mut writer = ZipWriter::new(BufWriter::new(file));
+    writer.start_file(
+        src_path.file_name().unwrap().to_str().unwrap(),
+        FileOptions::default()
+            .last_modified_time(
+                DateTime::try_from(OffsetDateTime::from(
+                    std::fs::metadata(src_path)?.modified()?,
+                ))
+                .unwrap(),
+            )
+            .unix_permissions(0o755)
+            .compression_method(zip::CompressionMethod::Deflated)
+            .compression_level(Some(9)),
+    )?;
+    let mut input = io::BufReader::new(File::open(src_path)?);
+    io::copy(&mut input, &mut writer)?;
+    if let Some(symbols_path) = symbols_path {
+        writer.start_file(
+            symbols_path.file_name().unwrap().to_str().unwrap(),
+            FileOptions::default()
+                .last_modified_time(
+                    DateTime::try_from(OffsetDateTime::from(
+                        std::fs::metadata(src_path)?.modified()?,
+                    ))
+                    .unwrap(),
+                )
+                .compression_method(zip::CompressionMethod::Deflated)
+                .compression_level(Some(9)),
+        )?;
+        let mut input = io::BufReader::new(File::open(symbols_path)?);
+        io::copy(&mut input, &mut writer)?;
+    }
+    writer.finish()?;
+    Ok(())
+}
+
+struct Target {
+    name: String,
+    server_path: PathBuf,
+    symbols_path: Option<PathBuf>,
+    artifact_name: String,
+}
+
+impl Target {
+    fn get(project_root: &Path) -> Self {
+        let name = match env::var("LLM_LS_TARGET") {
+            Ok(target) => target,
+            _ => {
+                if cfg!(target_os = "linux") {
+                    "x86_64-unknown-linux-gnu".to_string()
+                } else if cfg!(target_os = "windows") {
+                    "x86_64-pc-windows-msvc".to_string()
+                } else if cfg!(target_os = "macos") {
+                    "x86_64-apple-darwin".to_string()
+                } else {
+                    panic!("Unsupported OS, maybe try setting LLM_LS_TARGET")
+                }
+            }
+        };
+        let out_path = project_root.join("target").join(&name).join("release");
+        let (exe_suffix, symbols_path) = if name.contains("-windows-") {
+            (".exe".into(), Some(out_path.join("llm_ls.pdb")))
+        } else {
+            (String::new(), None)
+        };
+        let server_path = out_path.join(format!("llm-ls{exe_suffix}"));
+        let artifact_name = format!("llm-ls-{name}{exe_suffix}");
+        Self {
+            name,
+            server_path,
+            symbols_path,
+            artifact_name,
+        }
+    }
+}

xtask/src/flags.rs

@@ -0,0 +1,43 @@
+#![allow(unreachable_pub)]
+
+xflags::xflags! {
+    src "./src/flags.rs"
+
+    /// Run custom build command.
+    cmd xtask {
+        cmd dist {}
+    }
+}
+
+// generated start
+// The following code is generated by `xflags` macro.
+// Run `env UPDATE_XFLAGS=1 cargo build` to regenerate.
+#[derive(Debug)]
+pub struct Xtask {
+    pub subcommand: XtaskCmd,
+}
+
+#[derive(Debug)]
+pub enum XtaskCmd {
+    Dist(Dist),
+}
+
+#[derive(Debug)]
+pub struct Dist;
+
+impl Xtask {
+    #[allow(dead_code)]
+    pub fn from_env_or_exit() -> Self {
+        Self::from_env_or_exit_()
+    }
+
+    #[allow(dead_code)]
+    pub fn from_env() -> xflags::Result<Self> {
+        Self::from_env_()
+    }
+
+    #[allow(dead_code)]
+    pub fn from_vec(args: Vec<std::ffi::OsString>) -> xflags::Result<Self> {
+        Self::from_vec_(args)
+    }
+}

xtask/src/main.rs

@@ -0,0 +1,44 @@
+//! See <https://github.com/matklad/cargo-xtask/>.
+//!
+//! This binary defines various auxiliary build commands, which are not
+//! expressible with just `cargo`.
+//!
+//! This binary is integrated into the `cargo` command line by using an alias in
+//! `.cargo/config`.
+
+#![warn(
+    rust_2018_idioms,
+    unused_lifetimes,
+    semicolon_in_expressions_from_macros
+)]
+
+mod flags;
+
+mod dist;
+
+use std::{
+    env,
+    path::{Path, PathBuf},
+};
+use xshell::Shell;
+
+fn main() -> anyhow::Result<()> {
+    let flags = flags::Xtask::from_env_or_exit();
+
+    let sh = &Shell::new()?;
+    sh.change_dir(project_root());
+
+    match flags.subcommand {
+        flags::XtaskCmd::Dist(cmd) => cmd.run(sh),
+    }
+}
+
+fn project_root() -> PathBuf {
+    Path::new(
+        &env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| env!("CARGO_MANIFEST_DIR").to_owned()),
+    )
+    .ancestors()
+    .nth(1)
+    .unwrap()
+    .to_path_buf()
+}