Skip to content

Commit

Permalink
feat: provides correct semantic highlighting (#71)
Browse files Browse the repository at this point in the history 
* dev: initialize highlight test

* dev: add more cases

* fix: overlapping token

* fix: multiple lines token
  • Loading branch information
@Myriad-Dreamin
Myriad-Dreamin authored Mar 18, 2024
1 parent fb503ed commit 7e94c7d
Show file tree
Hide file tree
Showing 27 changed files with 375 additions and 149 deletions.
2 changes: 2 additions & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions crates/tinymist-query/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ once_cell.workspace = true
insta.workspace = true
serde.workspace = true
serde_json.workspace = true
sha2 = { version = "0.10" }
hex = { version = "0.4" }

# [lints]
# workspace = true
1 change: 1 addition & 0 deletions crates/tinymist-query/src/fixtures/semantic_tokens/base.typ
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
#let x = 1;
3 changes: 3 additions & 0 deletions crates/tinymist-query/src/fixtures/semantic_tokens/content-block.typ
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#[]
#[1]
#[#[1]#[#[1,2]]]
1 change: 1 addition & 0 deletions crates/tinymist-query/src/fixtures/semantic_tokens/fn.typ
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
#let f(x, y) = { x += y }
8 changes: 8 additions & 0 deletions crates/tinymist-query/src/fixtures/semantic_tokens/fn2.typ
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#let fn() = {
raw("1 + 2", lang: "typ")
```typ
1
2
```
[1]
}
1 change: 1 addition & 0 deletions crates/tinymist-query/src/fixtures/semantic_tokens/for-loop.typ
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
#for i in range(0) { i += 1 }
1 change: 1 addition & 0 deletions crates/tinymist-query/src/fixtures/semantic_tokens/heading.typ
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
=== #text("Text in heading")
6 changes: 6 additions & 0 deletions crates/tinymist-query/src/fixtures/semantic_tokens/snaps/[email protected]
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
source: crates/tinymist-query/src/semantic_tokens_full.rs
expression: "serde_json::to_string(&result).unwrap()"
input_file: crates/tinymist-query/src/fixtures/semantic_tokens/base.typ
---
{"data":[0,0,1,2,0,0,1,3,2,0,0,3,1,20,0,0,1,1,18,0,0,1,1,20,0,0,1,1,3,0,0,1,1,20,0,0,1,1,4,0,0,1,1,8,0]}
6 changes: 6 additions & 0 deletions crates/tinymist-query/src/fixtures/semantic_tokens/snaps/[email protected]
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
source: crates/tinymist-query/src/semantic_tokens_full.rs
expression: "serde_json::to_string(&result).unwrap()"
input_file: crates/tinymist-query/src/fixtures/semantic_tokens/content-block.typ
---
{"data":[0,0,1,8,0,0,1,1,8,0,0,1,1,8,0,0,1,2,20,0,1,0,0,20,0,0,0,1,8,0,0,1,1,8,0,0,1,1,20,0,0,1,1,8,0,0,1,2,20,0,1,0,0,20,0,0,0,1,8,0,0,1,1,8,0,0,1,1,8,0,0,1,1,8,0,0,1,1,20,0,0,1,1,8,0,0,1,1,8,0,0,1,1,8,0,0,1,1,8,0,0,1,1,8,0,0,1,3,20,0,0,3,1,8,0,0,1,1,8,0,0,1,1,8,0]}
6 changes: 6 additions & 0 deletions crates/tinymist-query/src/fixtures/semantic_tokens/snaps/[email protected]
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
source: crates/tinymist-query/src/semantic_tokens_full.rs
expression: "serde_json::to_string(&result).unwrap()"
input_file: crates/tinymist-query/src/fixtures/semantic_tokens/fn.typ
---
{"data":[0,0,1,2,0,0,1,3,2,0,0,3,1,20,0,0,1,1,5,0,0,1,1,8,0,0,1,1,18,0,0,1,1,8,0,0,1,1,20,0,0,1,1,18,0,0,1,1,8,0,0,1,1,20,0,0,1,1,3,0,0,1,1,20,0,0,1,1,8,0,0,1,1,20,0,0,1,1,18,0,0,1,1,20,0,0,1,2,3,0,0,2,1,20,0,0,1,1,18,0,0,1,1,20,0,0,1,1,8,0]}
6 changes: 6 additions & 0 deletions crates/tinymist-query/src/fixtures/semantic_tokens/snaps/[email protected]
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
source: crates/tinymist-query/src/semantic_tokens_full.rs
expression: "serde_json::to_string(&result).unwrap()"
input_file: crates/tinymist-query/src/fixtures/semantic_tokens/fn2.typ
---
{"data":[0,0,1,2,0,0,1,3,2,0,0,3,1,20,0,0,1,2,5,0,0,2,1,8,0,0,1,1,8,0,0,1,1,20,0,0,1,1,3,0,0,1,1,20,0,0,1,1,8,0,0,1,2,20,0,1,0,2,20,0,0,2,3,5,0,0,3,1,8,0,0,1,7,1,0,0,7,1,8,0,0,1,1,20,0,0,1,4,18,0,0,4,1,8,0,0,1,1,20,0,0,1,5,1,0,0,5,1,8,0,0,1,2,20,0,1,0,2,20,0,0,2,3,20,0,0,0,3,11,0,0,3,3,20,0,0,0,3,11,0,0,3,2,20,0,1,0,2,20,0,0,2,1,20,0,0,0,1,11,0,0,1,2,20,0,1,0,2,20,0,0,2,1,20,0,0,0,1,11,0,0,1,2,20,0,1,0,2,20,0,0,2,3,20,0,0,0,3,11,0,0,3,2,20,0,1,0,2,20,0,0,2,1,8,0,0,1,1,20,0,0,1,1,8,0,0,1,2,20,0,1,0,0,20,0,0,0,1,8,0]}
6 changes: 6 additions & 0 deletions crates/tinymist-query/src/fixtures/semantic_tokens/snaps/[email protected]
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
source: crates/tinymist-query/src/semantic_tokens_full.rs
expression: "serde_json::to_string(&result).unwrap()"
input_file: crates/tinymist-query/src/fixtures/semantic_tokens/for-loop.typ
---
{"data":[0,0,1,2,0,0,1,3,2,0,0,3,1,20,0,0,1,1,18,0,0,1,1,20,0,0,1,2,2,0,0,2,1,20,0,0,1,5,5,0,0,5,1,8,0,0,1,1,4,0,0,1,1,8,0,0,1,1,20,0,0,1,1,8,0,0,1,1,20,0,0,1,1,18,0,0,1,1,20,0,0,1,2,3,0,0,2,1,20,0,0,1,1,4,0,0,1,1,20,0,0,1,1,8,0]}
6 changes: 6 additions & 0 deletions crates/tinymist-query/src/fixtures/semantic_tokens/snaps/[email protected]
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
source: crates/tinymist-query/src/semantic_tokens_full.rs
expression: "serde_json::to_string(&result).unwrap()"
input_file: crates/tinymist-query/src/fixtures/semantic_tokens/heading.typ
---
{"data":[0,0,3,14,0,0,0,3,14,0,0,3,1,20,0,0,0,1,14,0,0,1,1,5,0,0,0,1,14,0,0,1,4,5,0,0,0,4,14,0,0,4,1,8,0,0,0,1,14,0,0,1,17,1,0,0,0,17,14,0,0,17,1,8,0,0,0,1,14,0]}
6 changes: 6 additions & 0 deletions crates/tinymist-query/src/fixtures/semantic_tokens/snaps/test@typst_lsp_issue_264.typ.snap
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
source: crates/tinymist-query/src/semantic_tokens_full.rs
expression: "serde_json::to_string(&result).unwrap()"
input_file: crates/tinymist-query/src/fixtures/semantic_tokens/typst_lsp_issue_264.typ
---
{"data":[0,0,4,0,0,1,0,5,0,0,1,0,4,0,0,1,0,2,0,0,0,2,2,20,0,1,0,2,20,0,1,0,0,20,0,0,0,3,20,0,0,0,3,11,0,0,3,4,20,0,0,0,4,11,0,0,4,2,20,0,1,0,0,20,0,0,0,17,20,0,0,0,17,11,0,0,17,2,20,0,1,0,0,20,0,0,0,3,20,0,0,0,3,11,0]}
6 changes: 6 additions & 0 deletions crates/tinymist-query/src/fixtures/semantic_tokens/snaps/test@typst_lsp_issue_401.typ.snap
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
source: crates/tinymist-query/src/semantic_tokens_full.rs
expression: "serde_json::to_string(&result).unwrap()"
input_file: crates/tinymist-query/src/fixtures/semantic_tokens/typst_lsp_issue_401.typ
---
{"data":[0,0,1,1,0,0,1,7,1,0,1,0,2,1,0,1,0,6,1,0]}
8 changes: 8 additions & 0 deletions crates/tinymist-query/src/fixtures/semantic_tokens/typst_lsp_issue_264.typ
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
/*
??
?
*/

```rust
let x: usize = 1;
```
3 changes: 3 additions & 0 deletions crates/tinymist-query/src/fixtures/semantic_tokens/typst_lsp_issue_401.typ
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#"line

break"
Loading

0 comments on commit 7e94c7d

Please sign in to comment.