Repository: clojure-lsp/clojure-lsp
Pull request #225 (2020-12-29 03:23:53)

[semantic-tokens] Add more semantic tokens
Adds more semantic tokens:

* declared function - `function` token
* declared variable - `function` token
* alias-reference -  `type`/`function` tokens

![image](https://user-images.githubusercontent.com/7820865/103256375-d8a5b980-496b-11eb-85db-b593ac7d07a6.png)



Patch:
diff --git a/README.md b/README.md
index c81797de..012a6bda 100644
--- a/README.md
+++ b/README.md
@@ -51,68 +51,68 @@ Bellow are all the currently supported LSP capabilities and their implementation
 <details>
 <summary><b>Supported LSP capabilities</b></summary>
 
-| capability                          | done | notes                                         |
-| ----------                          | ---- | -----                                         |
-| initialize                          | √    |                                               |
-| initialized                         | √    |                                               |
-| shutdown                            | √    |                                               |
-| exit                                | √    |                                               |
-| $/cancelRequest                     |      |                                               |
-| $/progress                          |      |                                               |
-| window/showMessage                  | √    |                                               |
-| window/showMessageRequest           |      |                                               |
-| window/logMessage                   |      |                                               |
-| window/workDoneProgress/create      |      |                                               |
-| window/workDoneProgress/cancel      |      |                                               |
-| telemetry/event                     |      |                                               |
-| client/registerCapability           | √    |                                               |
-| client/unregisterCapability         |      |                                               |
-| workspace/workspaceFolders          |      |                                               |
-| workspace/didChangeWorkspaceFolders |      |                                               |
-| workspace/didChangeConfiguration    | √    | Currently only log                            |
-| workspace/configuration             |      |                                               |
-| workspace/didChangeWatchedFiles     | √    |                                               |
-| workspace/symbol                    | √    |                                               |
-| workspace/executeCommand            | √    | See [Extra capabilities](#extra-capabilities) |
-| workspace/applyEdit                 | √    | TextDocumentEdit and RenameFile only          |
-| textDocument/didOpen                | √    |                                               |
-| textDocument/didChange              | √    |                                               |
-| textDocument/willSave               |      |                                               |
-| textDocument/willSaveWaitUntil      |      |                                               |
-| textDocument/didSave                | √    | Do nothing currently                          |
-| textDocument/didClose               | √    |                                               |
-| textDocument/publishDiagnostics     | √    |                                               |
-| textDocument/completion             | √    |                                               |
-| completionItem/resolve              | √    |                                               |
-| textDocument/hover                  | √    |                                               |
-| textDocument/signatureHelp          | X    | Implemented hard coded                        |
-| textDocument/declaration            |      |                                               |
-| textDocument/definition             | √    | TODO: Find java classes definition            |
-| textDocument/typeDefinition         |      |                                               |
-| textDocument/implementation         |      |                                               |
-| textDocument/references             | √    |                                               |
-| textDocument/documentHighlight      | √    |                                               |
-| textDocument/documentSymbol         | √    |                                               |
-| textDocument/codeAction             | √    |                                               |
-| textDocument/codeLens               | √    |                                               |
-| codeLens/resolve                    | √    |                                               |
-| textDocument/documentLink           |      |                                               |
-| documentLink/resolve                |      |                                               |
-| textDocument/documentColor          |      |                                               |
-| textDocument/colorPresentation      |      |                                               |
-| textDocument/formatting             | √    |                                               |
-| textDocument/rangeFormatting        | √    |                                               |
-| textDocument/onTypeFormatting       |      |                                               |
-| textDocument/rename                 | √    |                                               |
-| textDocument/prepareRename          |      |                                               |
-| textDocument/foldingRange           |      |                                               |
-| textDocument/selectionRange         |      |                                               |
-| textDocument/semanticTokens/full    | √    | Just `functions` and `macros` ATM             |
-| textDocument/semanticTokens/full/delta |     |                                             |
-| textDocument/semanticTokens/range   | √    |                                                |
-| workspace/semanticTokens/refresh   |     |                                                 |
-| textDocument/linkedEditingRange    |     |                                                 |
-| textDocument/moniker               |     |                                                 |
+| capability                             | done | notes                                         |
+| ----------                             | ---- | -----                                         |
+| initialize                             | √    |                                               |
+| initialized                            | √    |                                               |
+| shutdown                               | √    |                                               |
+| exit                                   | √    |                                               |
+| $/cancelRequest                        |      |                                               |
+| $/progress                             |      |                                               |
+| window/showMessage                     | √    |                                               |
+| window/showMessageRequest              |      |                                               |
+| window/logMessage                      |      |                                               |
+| window/workDoneProgress/create         |      |                                               |
+| window/workDoneProgress/cancel         |      |                                               |
+| telemetry/event                        |      |                                               |
+| client/registerCapability              | √    |                                               |
+| client/unregisterCapability            |      |                                               |
+| workspace/workspaceFolders             |      |                                               |
+| workspace/didChangeWorkspaceFolders    |      |                                               |
+| workspace/didChangeConfiguration       | √    | Currently only log                            |
+| workspace/configuration                |      |                                               |
+| workspace/didChangeWatchedFiles        | √    |                                               |
+| workspace/symbol                       | √    |                                               |
+| workspace/executeCommand               | √    | See [Extra capabilities](#extra-capabilities) |
+| workspace/applyEdit                    | √    | TextDocumentEdit and RenameFile only          |
+| textDocument/didOpen                   | √    |                                               |
+| textDocument/didChange                 | √    |                                               |
+| textDocument/willSave                  |      |                                               |
+| textDocument/willSaveWaitUntil         |      |                                               |
+| textDocument/didSave                   | √    | Do nothing currently                          |
+| textDocument/didClose                  | √    |                                               |
+| textDocument/publishDiagnostics        | √    |                                               |
+| textDocument/completion                | √    |                                               |
+| completionItem/resolve                 | √    |                                               |
+| textDocument/hover                     | √    |                                               |
+| textDocument/signatureHelp             | X    | Implemented hard coded                        |
+| textDocument/declaration               |      |                                               |
+| textDocument/definition                | √    | TODO: Find java classes definition            |
+| textDocument/typeDefinition            |      |                                               |
+| textDocument/implementation            |      |                                               |
+| textDocument/references                | √    |                                               |
+| textDocument/documentHighlight         | √    |                                               |
+| textDocument/documentSymbol            | √    |                                               |
+| textDocument/codeAction                | √    |                                               |
+| textDocument/codeLens                  | √    |                                               |
+| codeLens/resolve                       | √    |                                               |
+| textDocument/documentLink              |      |                                               |
+| documentLink/resolve                   |      |                                               |
+| textDocument/documentColor             |      |                                               |
+| textDocument/colorPresentation         |      |                                               |
+| textDocument/formatting                | √    |                                               |
+| textDocument/rangeFormatting           | √    |                                               |
+| textDocument/onTypeFormatting          |      |                                               |
+| textDocument/rename                    | √    |                                               |
+| textDocument/prepareRename             |      |                                               |
+| textDocument/foldingRange              |      |                                               |
+| textDocument/selectionRange            |      |                                               |
+| textDocument/semanticTokens/full       | √    | Just `functions`, type' and `macros` ATM             |
+| textDocument/semanticTokens/full/delta |      |                                               |
+| textDocument/semanticTokens/range      | √    |                                               |
+| workspace/semanticTokens/refresh       |      |                                               |
+| textDocument/linkedEditingRange        |      |                                               |
+| textDocument/moniker                   |      |                                               |
 </details>
 
 ## Extra capabilities
@@ -160,6 +160,8 @@ It is possible to pass some options to clojure-lsp through clients' `Initializat
 
 `keep-require-at-start?` if true, will keep first require at the first line instead of inserting a new line before it.
 
+`semantic-tokens?` if true or not present, will enable LSP semantic tokens server support for syntax highlighting. 
+
 `show-docs-arity-on-same-line?` if true, will keep the arity on the same line of the function on hover, useful for Emacs users.
 
 `auto-add-ns-to-new-files?` if true or not present, will automatically add the `ns` form in new files. 
diff --git a/src/clojure_lsp/feature/semantic_tokens.clj b/src/clojure_lsp/feature/semantic_tokens.clj
index eeb9a2d6..78f682a8 100644
--- a/src/clojure_lsp/feature/semantic_tokens.clj
+++ b/src/clojure_lsp/feature/semantic_tokens.clj
@@ -1,7 +1,10 @@
-(ns clojure-lsp.feature.semantic-tokens)
+(ns clojure-lsp.feature.semantic-tokens
+  (:require
+    [clojure.string :as string]))
 
 (def token-types
-  [:function
+  [:type
+   :function
    :macro])
 
 (def token-types-str
@@ -29,6 +32,14 @@
    (.indexOf token-types token-type)
    token-modifier])
 
+(defn ^:private alias-reference-usage->absolute-token
+  [{:keys [row col end-col str]}]
+  (let [slash-col (string/index-of str "/")
+        function-col (+ col (inc slash-col))
+        alias-end-col (+ col slash-col)]
+    [(usage->absolute-token {:row row :col col :end-col alias-end-col} :type)
+     (usage->absolute-token {:row row :col function-col :end-col end-col} :function)]))
+
 (defn ^:private usages->absolute-tokens
   [usages]
   (->> usages
@@ -37,11 +48,18 @@
          (fn [{:keys [tags] :as usage}]
            (cond
              (contains? tags :macro)
-             (usage->absolute-token usage :macro)
+             [(usage->absolute-token usage :macro)]
+
+             (contains? tags :declared)
+             [(usage->absolute-token usage :function)]
 
              (contains? tags :refered)
-             (usage->absolute-token usage :function))))
-       (remove nil?)))
+             [(usage->absolute-token usage :function)]
+
+             (contains? tags :alias-reference)
+             (alias-reference-usage->absolute-token usage))))
+       (remove nil?)
+       (mapcat identity)))
 
 (defn ^:private absolute-token->relative-token
   [tokens
diff --git a/src/clojure_lsp/main.clj b/src/clojure_lsp/main.clj
index 3a270276..5e803ae7 100644
--- a/src/clojure_lsp/main.clj
+++ b/src/clojure_lsp/main.clj
@@ -435,7 +435,8 @@
                                        (.setDocumentSymbolProvider true)
                                        (.setDocumentHighlightProvider true)
                                        (.setWorkspaceSymbolProvider true)
-                                       (.setSemanticTokensProvider (when (:semantic-tokens? settings)
+                                       (.setSemanticTokensProvider (when (or (not (contains? settings :semantic-tokens?))
+                                                                             (:semantic-tokens? settings))
                                                                      (doto (SemanticTokensWithRegistrationOptions.)
                                                                        (.setLegend (doto (SemanticTokensLegend.
                                                                                            semantic-tokens/token-types-str
diff --git a/src/clojure_lsp/parser.clj b/src/clojure_lsp/parser.clj
index c7c510da..51fa0784 100644
--- a/src/clojure_lsp/parser.clj
+++ b/src/clojure_lsp/parser.clj
@@ -165,11 +165,11 @@
       (-> (if-not ident-ns
             (cond
               java-sym {:sym java-sym :tags #{:norename :java}}
-              declaration? {:sym (ctr (name (or ns 'user)) ident-name)}
-              (and (keyword? ident) (= prefix "::")) {:sym (ctr (name ns) ident-name)}
-              (keyword? ident) {:sym ident}
+              declaration? {:sym (ctr (name (or ns 'user)) ident-name) :tags #{:declarion}}
+              (and (keyword? ident) (= prefix "::")) {:sym (ctr (name ns) ident-name) :tags #{:keyword :namespaced}}
+              (keyword? ident) {:sym ident :tags #{:keyword}}
               (contains? scoped ident) {:sym (ctr (name (get-in scoped [ident :ns])) ident-name) :tags #{:scoped}}
-              declared {:sym declared}
+              declared {:sym declared :tags #{:declared}}
               refered {:sym refered :tags #{:refered}}
               (contains? refer-all-syms ident) {:sym (get refer-all-syms ident)}
               core-clj? {:sym core-clj-symbol :tags #{:norename :core}}
@@ -181,16 +181,16 @@
                    (or
                      (and (keyword? ident) (= prefix "::"))
                      (symbol? ident)))
-              {:sym (ctr (name alias-ns) ident-name)}
+              {:sym (ctr (name alias-ns) ident-name) :tags #{:alias-reference}}
 
               (and (keyword? ident) (= prefix "::"))
               {:sym (ctr (name (gensym)) ident-name) :tags #{:unknown} :unknown-ns ident-ns}
 
               (keyword? ident)
-              {:sym ident}
+              {:sym ident :tags #{:keyword}}
 
               required-ns
-              {:sym ident}
+              {:sym ident :tags #{:raw-reference}}
 
               (contains? imports ident-ns)
               {:sym (symbol (name (get imports ident-ns)) ident-name) :tags #{:java :method :norename}}


Test patch:
diff --git a/test/clojure_lsp/features/semantic_tokens_test.clj b/test/clojure_lsp/features/semantic_tokens_test.clj
index 4fa96847..ce4a9b53 100644
--- a/test/clojure_lsp/features/semantic_tokens_test.clj
+++ b/test/clojure_lsp/features/semantic_tokens_test.clj
@@ -49,57 +49,87 @@
 (defn long-str [& strings] (clojure.string/join "\n" strings))
 
 (deftest usage->absolute-token
-  (is (= [6 3 3 0 -1]
+  (is (= [6 3 3 1 -1]
          (#'semantic-tokens/usage->absolute-token refered-usage-a
                                                   :function))))
 
 (deftest absolute-token->relative-token
   (testing "without previous token"
-    (is (= [6 3 3 0 -1]
+    (is (= [6 3 3 1 -1]
            (#'semantic-tokens/absolute-token->relative-token refered-tokens
                                                              0
                                                              (->token refered-usage-a :function)))))
   (testing "same line token"
-    (is (= [0 7 3 0 -1]
+    (is (= [0 7 3 1 -1]
            (#'semantic-tokens/absolute-token->relative-token refered-tokens
                                                              1
                                                              (->token refered-usage-b :function)))))
 
   (testing "other line token"
-    (is (= [2 2 3 0 -1]
+    (is (= [2 2 3 1 -1]
            (#'semantic-tokens/absolute-token->relative-token refered-tokens
                                                              2
                                                              (->token refered-usage-c :function))))))
 
+(deftest alias-reference-usage->absolute-token
+  (is (= [[19 4 2 0 -1] [19 7 4 1 -1]]
+         (#'semantic-tokens/alias-reference-usage->absolute-token
+          {:tags #{:alias-reference}
+           :end-row 20
+           :sym 'foo.bar/abcd
+           :str "ba/abcd"
+           :file-type #{:clj}
+           :col 5
+           :end-col 12
+           :row 20}))))
+
 (deftest full-tokens
   (testing "testing tokens order"
-    (let [code (long-str "(ns some.ns (:require [foo.bar :refer [baz]]))"
-                         "(def bla baz)"
+    (let [code (long-str "(ns some.ns"
+                         "  (:require [foo.bar :as f :refer [baz]]))"
+                         ""
+                         "(def bla 2)"
                          "baz"
-                         "(comment)"
-                         "baz")
+                         "f/bar"
+                         ""
+                         "bla"
+                         "(comment)")
           usages (parser/find-usages code :clj {})]
-      (is (= [1 9 3 0 -1
-              1 0 3 0 -1
-              1 1 7 1 -1
-              1 0 3 0 -1]
+      (is (= [4 0 3 1 -1
+              1 0 1 0 -1
+              0 2 3 1 -1
+              2 0 3 1 -1
+              1 1 7 2 -1]
              (semantic-tokens/full-tokens usages)))))
   (testing "testing user refered tokens"
     (let [code (long-str "(ns some.ns (:require [foo.bar :refer [baz]]))"
                          "(def bla baz)")
           usages (parser/find-usages code :clj {})]
-      (is (= [1 9 3 0 -1]
+      (is (= [1 9 3 1 -1]
              (semantic-tokens/full-tokens usages)))))
   (testing "testing macro refered tokens"
     (let [code (long-str "(ns some.ns (:require [clojure.test :refer [deftest]]))"
                          "(deftest some-test 1)")
           usages (parser/find-usages code :clj {})]
-      (is (= [1 1 7 1 -1]
+      (is (= [1 1 7 2 -1]
              (semantic-tokens/full-tokens usages)))))
   (testing "testing macro core tokens"
     (let [code (long-str "(comment 1)")
           usages (parser/find-usages code :clj {})]
-      (is (= [0 1 7 1 -1]
+      (is (= [0 1 7 2 -1]
+             (semantic-tokens/full-tokens usages)))))
+  (testing "testing function declared tokens"
+    (let [code (long-str "(def foo 1)"
+                         "foo")
+          usages (parser/find-usages code :clj {})]
+      (is (= [1 0 3 1 -1]
+             (semantic-tokens/full-tokens usages)))))
+  (testing "testing type alias for function tokens"
+    (let [code (long-str "(ns some.ns (:require [foo.bar :as fb]))"
+                         "fb/some-foo-bar")
+          usages (parser/find-usages code :clj {})]
+      (is (= [1 0 2 0 -1
+              0 3 12 1 -1]
              (semantic-tokens/full-tokens usages))))))
 
 (deftest range-tokens
@@ -111,6 +141,6 @@
                          "baz")
           range {:row 3 :col 0 :end-row 4 :end-col 0}
           usages (parser/find-usages code :clj {})]
-      (is (= [2 0 3 0 -1
-              1 1 7 1 -1]
+      (is (= [2 0 3 1 -1
+              1 1 7 2 -1]
              (semantic-tokens/range-tokens usages range))))))
diff --git a/test/clojure_lsp/parser_test.clj b/test/clojure_lsp/parser_test.clj
index b333c59f..7f8a9f64 100644
--- a/test/clojure_lsp/parser_test.clj
+++ b/test/clojure_lsp/parser_test.clj
@@ -2,7 +2,7 @@
   (:require
     [clojure-lsp.db :as db]
     [clojure-lsp.parser :as parser]
-    [clojure.test :refer :all]
+    [clojure.test :refer [deftest is testing]]
     [clojure.tools.logging :as log]
     [rewrite-clj.zip :as z]))
 
@@ -60,14 +60,14 @@
           context (volatile! {})]
       (parser/handle-defn (z/next zloc) zloc context {} false)
       (is (= '{"foo" #{:declare :public}
-               "bar" #{}}
+               "bar" #{:keyword}}
              (into {} (map (juxt (comp name :sym) :tags) (:usages @context)))))))
   (testing "defn without meta caret"
     (let [zloc (z/of-string "(defn foo {:bar true} [] 1)")
           context (volatile! {})]
       (parser/handle-defn (z/next zloc) zloc context {} false)
       (is (= '{"foo" #{:declare :public}
-               "bar" #{}}
+               "bar" #{:keyword}}
              (into {} (map (juxt (comp name :sym) :tags) (:usages @context))))))))
 
 (deftest qualify-ident-test
@@ -77,7 +77,7 @@
     (is (= {:sym 'java.lang.Exception :tags #{:java :norename} :str "Exception."} (parser/qualify-ident 'Exception. context {} false)))
     (is (= {:sym '.getMessage :tags #{:java :method :norename} :str ".getMessage"} (parser/qualify-ident '.getMessage context {} false)))
     (is (= {:sym 'java.lang.Thread/sleep :tags #{:method :norename :java} :str "Thread/sleep"} (parser/qualify-ident 'Thread/sleep context {} false)))
-    (is (= {:sym ':foo :str ":foo" :tags #{}} (parser/qualify-ident :foo context {} false)))
+    (is (= {:sym ':foo :str ":foo" :tags #{:keyword}} (parser/qualify-ident :foo context {} false)))
     (is (= {:sym 'clojure.core/for :tags #{:norename :core} :str "for"} (parser/qualify-ident 'for context {} false)))
     (is (= {:sym 'foo.bar/baz :tags #{:refered} :str "baz"}
            (parser/qualify-ident 'baz (merge context {:refers {"baz" 'foo.bar/baz}}) {} false)))
@@ -270,7 +270,7 @@
       (is (= [":foo/foo" ":foo" "::foo" "::q/foo" "::x/foo"] (mapv :str usages)))
       (is (= [:foo/foo :foo :bar/foo :qux/foo] (butlast (mapv :sym usages))))
       (is (= :foo (:sym (second usages))))
-      (is (= [#{} #{} #{} #{}] (seq (remove nil? (mapv :tags (butlast usages))))))
+      (is (= [#{:keyword} #{:keyword} #{:namespaced :keyword} #{:alias-reference}] (seq (remove nil? (mapv :tags (butlast usages))))))
       (is (not= 'x (namespace (:sym (last usages)))))
       (is (not= 'user (namespace (:sym (last usages)))))
       (is (= #{:unknown} (:tags (last usages))))))
@@ -280,7 +280,7 @@
           [a1 a2 b1 b2 c1 c2 d1 d2 e1 e2 f1 f2 :as usages] (drop 5 (parser/find-usages code :clj {}))]
       (is (= [":foo/a" ":foo/a" ":b" ":b" "::c" "::c" "d" "d" "::q/e" "::q/e" "::x/f" "::x/f"] (mapv :str usages)))
       (is (= [:foo/a :b :bar/c :d :qux/e] (map :sym [a1 b1 c1 d1 e1])))
-      (is (= [#{} #{} #{} #{}] (seq (remove nil? (mapv :tags [a1 b1 c1 d1 e1])))))
+      (is (= [#{:keyword} #{:keyword} #{:namespaced :keyword} #{:alias-reference}] (seq (remove nil? (mapv :tags [a1 b1 c1 d1 e1])))))
       (is (not= 'x (namespace (:sym (last usages)))))
       (is (not= 'user (namespace (:sym (last usages)))))
       (is (= #{:unknown} (:tags f1)))