Improve ShaderLab compilation error log and package build

.github/workflows/ci.yml

@@ -50,7 +50,7 @@ jobs:
 
       - name: Install
         run: pnpm install
-      - run: npm run build:editor
+      - run: npm run build
 
   codecov:
     runs-on: macos-latest
@@ -66,7 +66,7 @@ jobs:
       - name: Install
         run: pnpm install
       - name: Build
-        run: npm run build:editor
+        run: npm run build
       - name: Test
         run: npm run test-cov
       - run: pnpm install codecov -w
@@ -94,7 +94,7 @@ jobs:
       - name: Run Cypress Tests
         uses: cypress-io/github-action@v5
         with:
-          build: npm run build:editor
+          build: npm run build
           start: npm run e2e:case
           wait-on: "http://localhost:5175"
           wait-on-timeout: 120

package.json

@@ -12,16 +12,12 @@
     "build:editor": "npm run b:module:editor && npm run b:types",
     "lint": "eslint packages/*/src --ext .ts",
     "watch": "cross-env NODE_ENV=release BUILD_TYPE=MODULE rollup -cw -m inline",
-    "watch:editor": "cross-env NODE_ENV=editor BUILD_TYPE=MODULE rollup -cw -m inline",
     "watch:umd": "cross-env NODE_ENV=release BUILD_TYPE=UMD rollup -cw -m inline",
-    "watch:umd:editor": "cross-env NODE_ENV=editor BUILD_TYPE=UMD rollup -cw -m inline",
     "b:types": "pnpm -r --filter=./packages/* run b:types",
     "b:module": "cross-env BUILD_TYPE=MODULE NODE_ENV=release rollup -c",
-    "b:module:editor": "cross-env BUILD_TYPE=MODULE NODE_ENV=editor rollup -c",
     "b:umd": "cross-env BUILD_TYPE=UMD NODE_ENV=release rollup -c",
     "b:miniprogram": "cross-env BUILD_TYPE=MINI rollup -c",
     "b:all": "cross-env NODE_ENV=release npm run b:types && cross-env BUILD_TYPE=ALL NODE_ENV=release rollup -c",
-    "b:all:editor": "cross-env NODE_ENV=editor npm run b:types && cross-env BUILD_TYPE=ALL rollup -c",
     "clean": "pnpm -r exec rm -rf dist && pnpm -r exec rm -rf types",
     "e2e:case": "pnpm -C ./e2e run case",
     "e2e": "cypress run --browser chrome --headless",

packages/shader-lab/README.md

@@ -26,6 +26,19 @@ const shader = Shader.create(galaceanShaderCode);
 engine.run()
 ```
 
+There are two versions of ShaderLab: `Release` and `Debug`. The Debug version offers more user-friendly diagnostic information for debug ShaderLab compilation errors, while the Release version provides superior performance.
+
+you can use debug version by import:
+
+```ts
+// umd
+import { ShaderLab } from "@galacean/engine-shader-lab/dist/browser.editor";
+// esmoudle
+import { ShaderLab } from "@galacean/engine-shader-lab/dist/module.editor";
+// commonjs
+import { ShaderLab } from "@galacean/engine-shader-lab/dist/main.editor";
+```
+
 ## CFG Grammar conflict detection
 
 The Galacean ShaderLab syntax is defined using Context-Free Grammar (CFG) and is documented within the `\*.y` file. When modifications to the ShaderLab syntax are required, it is recommended to make changes to the existing CFG syntax file, and employ [Bison](https://www.gnu.org/software/bison/manual/bison.html) to detect any potential grammar conflicts.

packages/shader-lab/src/Error.ts

@@ -1,19 +1,77 @@
 // #if _EDITOR
-import { ShaderRange } from "./common";
+import { Logger } from "@galacean/engine";
+import { ShaderPosition, ShaderRange } from "./common";
 
 export abstract class GSError extends Error {
-  readonly loc: ShaderRange;
+  static wrappingLineCount = 2;
 
-  constructor(message: string, loc: ShaderRange, cause?: Error) {
+  readonly loc: ShaderRange | ShaderPosition;
+  readonly source: string;
+  readonly file?: string;
+
+  constructor(message: string, loc: ShaderRange | ShaderPosition, source: string, file?: string, cause?: Error) {
     super(message, { cause });
     this.loc = loc;
+    this.source = source;
+    this.file = file;
+  }
+
+  log(_source?: string): void {
+    if (!Logger.enable) return;
+    let start: ShaderPosition, end: ShaderPosition;
+    const { message, loc, source: originSource } = this;
+    let source = originSource;
+    if (_source) source = _source;
+    if (!source) {
+      Logger.error(message);
+      return;
+    }
+
+    if (loc instanceof ShaderPosition) {
+      start = end = loc;
+    } else {
+      start = loc.start;
+      end = loc.end;
+    }
+    const lines = source.split("\n");
+
+    let diagnosticMessage = `${this.name}: ${message}\n\n`;
+    const lineSplit = "|···";
+
+    for (let i = 0; i <= end.line + GSError.wrappingLineCount; i++) {
+      if (i < start.line - GSError.wrappingLineCount) continue;
+
+      diagnosticMessage += lineSplit;
+      diagnosticMessage += `${lines[i]}\n`;
+      if (i >= start.line && i <= end.line) {
+        diagnosticMessage += " ".repeat(lineSplit.length + start.column);
+        diagnosticMessage += "^".repeat(Math.max(end.column - start.column, 1));
+        diagnosticMessage += "\n";
+      }
+    }
+
+    Logger.error(diagnosticMessage);
+  }
+}
+
+export class PreprocessorError extends GSError {
+  constructor(message: string, loc: ShaderRange | ShaderPosition, source: string, file?: string, cause?: Error) {
+    super(message, loc, source, file, cause);
+    this.name = "PreprocessorError";
+  }
+}
+
+export class CompilationError extends GSError {
+  constructor(message: string, loc: ShaderRange | ShaderPosition, source: string, file?: string, cause?: Error) {
+    super(message, loc, source, file, cause);
+    this.name = "CompilationError";
   }
 }
 
-export class SemanticError extends GSError {
-  constructor(message: string, loc: ShaderRange, cause?: Error) {
-    super(message, loc, cause);
-    this.name = "SemanticError";
+export class ScannerError extends GSError {
+  constructor(message: string, loc: ShaderRange | ShaderPosition, source: string, file?: string, cause?: Error) {
+    super(message, loc, source, file, cause);
+    this.name = "ScannerError";
   }
 }
 // #endif

packages/shader-lab/src/ShaderLab.ts

@@ -2,42 +2,30 @@ import { Lexer } from "./lexer";
 import { ShaderTargetParser } from "./parser";
 import { Preprocessor } from "./preprocessor";
 import { GLES100Visitor, GLES300Visitor } from "./codeGen";
-import { IShaderContent, IShaderLab } from "@galacean/engine-design/src/shader-lab";
+import { IShaderContent, IShaderLab } from "@galacean/engine-design";
 import { ShaderContentParser } from "./contentParser";
 // @ts-ignore
 import { Logger, ShaderLib, ShaderMacro, ShaderPass, ShaderPlatformTarget } from "@galacean/engine";
 import { ShaderPosition, ShaderRange } from "./common";
 import { ShaderLabObjectPool } from "./ShaderLabObjectPool";
+// #if _EDITOR
+import { GSError } from "./Error";
+// #endif
+import { PpParser } from "./preprocessor/PpParser";
 
 export class ShaderLab implements IShaderLab {
-  /**
-   * @internal
-   */
   private static _parser = ShaderTargetParser.create();
-  /**
-   * @internal
-   */
   private static _shaderPositionPool = new ShaderLabObjectPool(ShaderPosition);
+  private static _shaderRangePool = new ShaderLabObjectPool(ShaderRange);
+
   /**
    * @internal
    */
-  private static _shaderRangePool = new ShaderLabObjectPool(ShaderRange);
+  static _processingPassText?: string;
 
-  static createPosition(
-    index: number,
-    // #if _EDITOR
-    line?: number,
-    column?: number
-    // #endif
-  ): ShaderPosition {
+  static createPosition(index: number, line?: number, column?: number): ShaderPosition {
     const position = this._shaderPositionPool.get();
-    position.setX(
-      index,
-      // #if _EDITOR
-      line,
-      column
-      // #endif
-    );
+    position.setX(index, line, column);
     return position;
   }
 
@@ -47,6 +35,17 @@ export class ShaderLab implements IShaderLab {
     return range;
   }
 
+  // #if _EDITOR
+  private _errors: GSError[] = [];
+
+  /**
+   * Retrieve the compilation errors
+   */
+  get errors(): GSError[] | undefined {
+    return this._errors;
+  }
+  // #endif
+
   _parseShaderPass(
     source: string,
     vertexEntry: string,
@@ -56,7 +55,6 @@ export class ShaderLab implements IShaderLab {
     platformMacros: string[],
     basePathForIncludeKey: string
   ) {
-    ShaderLabObjectPool.clearAllShaderLabObjectPool();
     Preprocessor.reset(ShaderLib, basePathForIncludeKey);
     for (const macro of macros) {
       Preprocessor.addPredefinedMacro(macro.name, macro.value);
@@ -66,36 +64,67 @@ export class ShaderLab implements IShaderLab {
       Preprocessor.addPredefinedMacro(platformMacros[i]);
     }
 
-    // #if _EDITOR
-    // TODO: index to position
-    // Logger.convertSourceIndex = Preprocessor.convertSourceIndex.bind(Preprocessor);
-    // #endif
-
     const preprocessorStart = performance.now();
-
     const ppdContent = Preprocessor.process(source);
+    // #if _EDITOR
+    if (PpParser._errors.length > 0) {
+      for (const err of PpParser._errors) {
+        this._errors.push(<GSError>err);
+      }
+      this._logErrors(this._errors);
+      return { vertex: "", fragment: "" };
+    }
+    // #endif
 
     Logger.info(`[pass compilation - preprocessor]  cost time ${performance.now() - preprocessorStart}ms`);
 
     const lexer = new Lexer(ppdContent);
     const tokens = lexer.tokenize();
-    const program = ShaderLab._parser.parse(tokens);
+
+    const { _parser: parser } = ShaderLab;
+
+    ShaderLab._processingPassText = ppdContent;
+    const program = parser.parse(tokens);
+
+    // #if _EDITOR
+    for (const err of parser.errors) {
+      this._errors.push(err);
+    }
     if (!program) {
+      this._logErrors(this._errors);
       return { vertex: "", fragment: "" };
     }
+    // #endif
+
     const codeGen =
       backend === ShaderPlatformTarget.GLES100 ? GLES100Visitor.getVisitor() : GLES300Visitor.getVisitor();
 
     const start = performance.now();
     const ret = codeGen.visitShaderProgram(program, vertexEntry, fragmentEntry);
     Logger.info(`[CodeGen] cost time: ${performance.now() - start}ms`);
+    ShaderLab._processingPassText = undefined;
+
+    // #if _EDITOR
+    for (const err of codeGen.errors) {
+      this._errors.push(err);
+    }
+    this._logErrors(this._errors);
+    // #endif
 
     return ret;
   }
 
   _parseShaderContent(shaderSource: string): IShaderContent {
+    ShaderLabObjectPool.clearAllShaderLabObjectPool();
     ShaderContentParser.reset();
-    return ShaderContentParser.parse(shaderSource);
+    // #if _EDITOR
+    this._errors.length = 0;
+    // #endif
+    const ret = ShaderContentParser.parse(shaderSource);
+    for (const error of ShaderContentParser._errors) {
+      this._errors.push(error);
+    }
+    return ret;
   }
 
   // #if _EDITOR
@@ -105,8 +134,8 @@ export class ShaderLab implements IShaderLab {
    */
   _parse(
     shaderSource: string,
-    macros: ShaderMacro[],
-    backend: ShaderPlatformTarget
+    macros: ShaderMacro[] = [],
+    backend: ShaderPlatformTarget = ShaderPlatformTarget.GLES100
   ): (ReturnType<ShaderLab["_parseShaderPass"]> & { name: string })[] {
     const structInfo = this._parseShaderContent(shaderSource);
     const passResult = [] as any;
@@ -129,5 +158,16 @@ export class ShaderLab implements IShaderLab {
     }
     return passResult;
   }
+
+  /**
+   * @internal
+   */
+  _logErrors(errors: GSError[], source?: string) {
+    if (errors.length === 0 || !Logger.isEnabled) return;
+    Logger.error(`${errors.length} errors occur!`);
+    for (const err of errors) {
+      err.log(source);
+    }
+  }
   // #endif
 }

packages/shader-lab/src/Utils.ts

@@ -1,11 +1,10 @@
 import { ENonTerminal, GrammarSymbol } from "./parser/GrammarSymbol";
 import { BaseToken as Token } from "./common/BaseToken";
-import { EKeyword, ETokenType, GalaceanDataType, ShaderRange, ShaderPosition } from "./common";
+import { EKeyword, ETokenType, GalaceanDataType } from "./common";
 import { TreeNode } from "./parser/AST";
 // #if _EDITOR
 import State from "./lalr/State";
 // #endif
-import { Logger } from "@galacean/engine";
 
 export class ParserUtils {
   static unwrapNodeByType<T = TreeNode>(node: TreeNode, type: ENonTerminal): T | undefined {
@@ -39,11 +38,6 @@ export class ParserUtils {
     return sm < ENonTerminal.START;
   }
 
-  static throw(pos: ShaderPosition | ShaderRange | number, ...msgs: any[]) {
-    Logger.error(pos.toString(), ...msgs);
-    throw msgs.join(" ");
-  }
-
   /**
    * @internal
    */