change tree-sitter to wasm (#2)

* init tree-sitter wasm

* 修复一些问题

* 修复一些问题

* update fetchModels

* update yml

Author: likaiqiang

.github/workflows/beta-pack.yml

@@ -20,13 +20,15 @@ jobs:
         with:
           submodules: recursive
       - name: Cache MKL
+        id: mkl
         uses: actions/cache@v3
         with:
           path: C:\Program Files (x86)\Intel\oneAPI\mkl\latest
           key: ${{ runner.os }}-mkl-caches-w_BaseKit_p_2023.1.0.47256_offline
           restore-keys: |
             ${{ runner.os }}-mkl-
       - name: Install MKL
+        if: steps.mkl.outputs.cache-hit != 'true'
         run: |
           echo "MKLROOT=C:\Program Files (x86)\Intel\oneAPI\mkl\latest" >> $GITHUB_ENV
           .github/workflows/install_mkl_windows.bat "https:/registrationcenter-download.intel.com/akdlm/IRC_NAS/c961e083-5685-4f0b-ada5-c6cf16f561dd/w_BaseKit_p_2023.1.0.47256_offline.exe" "intel.oneapi.win.mkl.devel"

.github/workflows/release.yml

@@ -21,13 +21,15 @@ jobs:
         with:
           submodules: recursive
       - name: Cache MKL
+        id: mkl
         uses: actions/cache@v3
         with:
           path: C:\Program Files (x86)\Intel\oneAPI\mkl\latest
           key: ${{ runner.os }}-mkl-caches-w_BaseKit_p_2023.1.0.47256_offline
           restore-keys: |
             ${{ runner.os }}-mkl-
       - name: Install MKL
+        if: steps.mkl.outputs.cache-hit != 'true'
         run: |
           echo "MKLROOT=C:\Program Files (x86)\Intel\oneAPI\mkl\latest" >> $GITHUB_ENV
           .github/workflows/install_mkl_windows.bat "https:/registrationcenter-download.intel.com/akdlm/IRC_NAS/c961e083-5685-4f0b-ada5-c6cf16f561dd/w_BaseKit_p_2023.1.0.47256_offline.exe" "intel.oneapi.win.mkl.devel"

build.js

@@ -1,47 +1,26 @@
 const { exec } = require('child_process');
-const fs = require('fs');
 const path = require('path');
-const tar = require('tar');
 const fsExtra = require('fs-extra')
+const os = require('os');
+const { rimrafSync } = require('rimraf')
 
-
-function buildLib({command, cwd, destination}){
+async function buildWasm({ languageName}){
+  const command = 'npx tree-sitter build-wasm .'
+  const cwd = path.join(__dirname, `./node_modules/tree-sitter-${languageName}`)
+  const destination = path.join(__dirname, './src/assets/wasm')
   return new Promise((resolve,reject)=>{
     exec(command, { cwd }, (error, stdout, stderr) => {
       if (error) {
         return reject(`执行的命令出错: ${error}`)
       }
 
-      console.log(`stdout: ${stdout}`);
-      console.error(`stderr: ${stderr}`);
-
-      // 找到生成的压缩包
-      const prebuildsDir = path.join(cwd, 'prebuilds');
-      fs.readdir(prebuildsDir, (err, files) => {
-        if (err) {
-          return reject(`读取目录出错: ${err}`)
-        }
-
-        // 找到 .tar.gz 文件
-        const tarFile = files.find(file => file.endsWith('.tar.gz'));
-        if (!tarFile) {
-          return reject('没有找到 .tar.gz 文件')
-        }
+      console.log(`stdout: tree-sitter-${languageName} generate success`);
 
-        // 解压缩文件
-        tar.x({
-          file: path.join(prebuildsDir, tarFile),
-          C: prebuildsDir,
-        }, err => {
-          if (err) {
-            return reject(`解压缩文件出错: ${err}`)
-          }
-          fsExtra.copy(
-            path.join(prebuildsDir,'build','Release'),
-            destination
-          ).then(resolve)
-        });
-      });
+      const wasmFile = `tree-sitter-${languageName}.wasm`
+      fsExtra.copy(
+        path.join(path.join(cwd, wasmFile)),
+        path.join(destination, wasmFile)
+      ).then(resolve)
     });
   })
 }
@@ -54,47 +33,43 @@ async function serialPrommise(taskFunc = []){
   },Promise.resolve())
 }
 
-const buildNodeModulesLib = ()=>{
-  return ['javascript','cpp','go','java','php','python','ruby','rust','scala','markdown','html','solidity','kotlin'].map(language=>{
-    return buildLib({
-      command: 'npx prebuild -r electron -t 26.0.0 --strip',
-      cwd: path.join(__dirname, 'node_modules', `tree-sitter-${language}`),
-      destination: path.join(__dirname, 'src', 'assets', 'node', `tree-sitter-${language}`)
-    })
-  })
-}
-
-const copyNodeTypes = ()=>{
-  return ['javascript','cpp','go','java','php','python','ruby','rust','scala','markdown','html','solidity','kotlin'].map(language=>{
+const buildFaiss = async function(){
+  const copyFaiss = async ()=>{
     return fsExtra.copy(
-      path.join(__dirname, 'node_modules', `tree-sitter-${language}`, 'src', 'node-types.json'),
-      path.join(__dirname, 'src', 'assets', 'node', `tree-sitter-${language}`, 'node-types.json')
+      path.join(__dirname, 'node_modules', 'faiss-node','build', 'Release'),
+      path.join(__dirname, 'src', 'assets', 'node', 'faiss-node', 'build')
     )
-  })
+  }
+  if(os.platform() === 'darwin'){
+    return new Promise((resolve, reject)=>{
+      rimrafSync(
+        path.join(__dirname,'node_modules','faiss-node','deps','faiss')
+      )
+      const command = 'git clone -b v1.7.4 --depth 1 https://github.com/facebookresearch/faiss.git deps/faiss && npm i cmake-js && npm run build'
+      exec(command, { cwd: path.join(__dirname,'node_modules','faiss-node') }, (error, stdout, stderr) => {
+        if (error) {
+          return reject(`执行的命令出错: ${error}`)
+        }
+        copyFaiss().then(resolve)
+      });
+    })
+  }
+  else {
+    await copyFaiss()
+  }
 }
 
-const taskFunc = [
-  ()=>{
-    return Promise.all([
-      buildLib({
-        command: 'npx prebuild -r electron -t 26.0.0 --strip',
-        cwd: path.join(__dirname, 'src', 'deps','tree-sitter'),
-        destination: path.join(__dirname, 'src', 'assets', 'node', 'tree-sitter')
-      }),
-      ...buildNodeModulesLib()
-    ])
-  },
-  ()=>{
-    return Promise.all([
-      fsExtra.copy(
-        path.join(__dirname, 'node_modules', 'faiss-node','build', 'Release'),
-        path.join(__dirname, 'src', 'assets', 'node', 'faiss-node', 'build')
-      ),
-      ...copyNodeTypes()
-    ])
-  }
-]
+// const taskFunc = [
+//   buildFaiss,
+//   ()=>{
+//     return Promise.all(
+//       ['javascript','cpp','go','java','php','python','ruby','rust','scala','markdown','html','solidity','kotlin'].map(language=> buildWasm({languageName: language}))
+//     )
+//   }
+// ]
+//
+// serialPrommise(taskFunc).then().catch(err=>{
+//   console.log(err);
+// })
 
-serialPrommise(taskFunc).then().catch(err=>{
-  console.log(err);
-})
+buildFaiss().then()