Repository: spyder-ide/spyder

Issue:
Create a get_words method in introspection utils
Create a generic function that finds all words in a given source code file (language agnostic)

here
https://github.com/spyder-ide/spyder/blob/master/spyder/utils/introspection/utils.py

To filter out letters or tokens we could use the information stored on pygments.

So that we can make inline completions on word type (like VS CODE does).

We need to write test for a couple of different files (c, C++, python, R, HTML, CSS)




Discussion:
But that's what the fallback completer is doing right now, isn't it?

Nope, not for EVERY word... only for special words of the language. Look at VS code

Hello @goanpeca  @ccordoba12 

I have been working on code completion widget.
- Now, I have a change for activating code completion  widget always a user input and suggestions exists (key event deferent to modifier events) #3708 
- An interesting thing is happening (three different behavioral on the widget) as you can see in the gif below

![bug-widgetcompletion](https://cloud.githubusercontent.com/assets/5307894/20452744/4bf93128-addf-11e6-8119-d5f383d58835.gif)

It can be associated with the three actual plugins  (Jedi, Rop, fallback ) **fight for completion task** for same user input text.

I'm planning to have a first PR with this changes.

I think that we are in a good way for having a better code completion feature, thanks for your fantastic suggestions @ccordoba12, @goanpeca 


Resolving patch:
diff --git a/spyder/utils/introspection/utils.py b/spyder/utils/introspection/utils.py
--- a/spyder/utils/introspection/utils.py
+++ b/spyder/utils/introspection/utils.py
@@ -126,7 +126,7 @@ def _get_info(self):
                 self.position = self.position - len(self.line) + self.column
 
     def _get_docstring(self):
-        """Find the docstring we are currently in"""
+        """Find the docstring we are currently in."""
         left = self.position
         while left:
             if self.source_code[left: left + 3] in ['"""', "'''"]:
@@ -150,7 +150,7 @@ def __eq__(self, other):
             return False
 
     def __getitem__(self, item):
-        """Allow dictionary-like access"""
+        """Allow dictionary-like access."""
         return getattr(self, item)
 
     def serialize(self):
@@ -210,11 +210,37 @@ def get_keywords(lexer):
                 continue
     return keywords
 
+def get_words(file_path=None, content=None, extension=None):
+    """
+    Extract all words from a source code file to be used in code completion.
+
+    Extract the list of words that contains the file in the editor,
+    to carry out the inline completion similar to VSCode.
+    """
+    if (file_path is None and (content is None or extension is None) or
+                    file_path and content and extension):
+        error_msg = ('Must provide `file_path` or `content` and `extension`')
+        raise Exception(error_msg)
+
+    if file_path and content is None and extension is None:
+        extension = os.path.splitext(file_path)[1]
+        with open(file_path) as infile:
+            content = infile.read()
+
+    if extension in ['.css']:
+        regex = re.compile(r'([^a-zA-Z-])')
+    elif extension in ['.R', '.c', 'md', '.cpp, java', '.py']:
+        regex = re.compile(r'([^a-zA-Z_])')
+    else:
+        regex = re.compile(r'([^a-zA-Z])')
+
+    words = sorted(set(regex.sub(r' ', content).split()))
+    return words
 
 @memoize
 def get_parent_until(path):
     """
-    Given a file path, determine the full module path
+    Given a file path, determine the full module path.
 
     e.g. '/usr/lib/python2.7/dist-packages/numpy/core/__init__.pyc' yields
     'numpy.core'


Test patch:
diff --git a/spyder/utils/tests/data/example.R b/spyder/utils/tests/data/example.R
new file mode 100644
--- /dev/null
+++ b/spyder/utils/tests/data/example.R
@@ -0,0 +1,3 @@
+hello <- function( name ) {
+    sprintf( "Hello, %s", name );
+}
diff --git a/spyder/utils/tests/data/example.c b/spyder/utils/tests/data/example.c
new file mode 100644
--- /dev/null
+++ b/spyder/utils/tests/data/example.c
@@ -0,0 +1,22 @@
+#include <stdio.h>
+
+int main()
+{
+  struct foo
+  {
+    int x;
+    float y;
+  };
+
+  struct foo var;
+  struct foo* pvar;
+  pvar = &var;
+
+  var.x = 5;
+  (&var)->y = 14.3;
+  printf("%i - %.02f\n", var.x, (&var)->y);
+  pvar->x = 6;
+  pvar->y = 22.4;
+  printf("%i - %.02f\n", pvar->x, pvar->y);
+  return 0;
+}
diff --git a/spyder/utils/tests/data/example.cpp b/spyder/utils/tests/data/example.cpp
new file mode 100644
--- /dev/null
+++ b/spyder/utils/tests/data/example.cpp
@@ -0,0 +1,74 @@
+#include <iostream>
+#include <vector>
+using namespace std;
+
+// Consider an actual class.
+class Obj {
+   static int i, j;
+	
+public:
+   void f() const { cout << i++ << endl; }
+   void g() const { cout << j++ << endl; }
+};
+
+// Static member definitions:
+int Obj::i = 10;
+int Obj::j = 12;
+
+// Implement a container for the above class
+class ObjContainer {
+   vector<Obj*> a;
+public:
+   void add(Obj* obj) { 
+      a.push_back(obj);  // call vector's standard method.
+   }
+   friend class SmartPointer;
+};
+
+// implement smart pointer to access member of Obj class.
+class SmartPointer {
+   ObjContainer oc;
+   int index;
+public:
+   SmartPointer(ObjContainer& objc) { 
+      oc = objc;
+      index = 0;
+   }
+	
+   // Return value indicates end of list:
+   bool operator++() // Prefix version { 
+      if(index >= oc.a.size()) return false;
+      if(oc.a[++index] == 0) return false;
+      return true;
+   }
+	
+   bool operator++(int) // Postfix version { 
+      return operator++();
+   }
+	
+   // overload operator->
+   Obj* operator->() const  {
+      if(!oc.a[index]) {
+         cout << "Zero value";
+         return (Obj*)0;
+      }
+      return oc.a[index];
+   }
+};
+
+int main() {
+   const int sz = 10;
+   Obj o[sz];
+   ObjContainer oc;
+	
+   for(int i = 0; i < sz; i++) {
+      oc.add(&o[i]);
+   }
+	
+   SmartPointer sp(oc); // Create an iterator
+   do {
+      sp->f(); // smart pointer call
+      sp->g();
+   } while(sp++);
+   return 0;
+}
diff --git a/spyder/utils/tests/data/example.css b/spyder/utils/tests/data/example.css
new file mode 100644
--- /dev/null
+++ b/spyder/utils/tests/data/example.css
@@ -0,0 +1,7 @@
+<style type="text/css">
+h1 {
+        color: DeepSkyBlue;
+}
+</style>
+
+<h1>Hello, world! nombre-valido </h1>
diff --git a/spyder/utils/tests/data/example.html b/spyder/utils/tests/data/example.html
new file mode 100644
--- /dev/null
+++ b/spyder/utils/tests/data/example.html
@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="utf-8">
+        <title>Hello World</title>
+    </head>
+    <body>
+        <h1>Hello World</h1>
+        <p>
+            Jamie was here.
+        </p>
+    </body>
+</html>
diff --git a/spyder/utils/tests/data/example.java b/spyder/utils/tests/data/example.java
new file mode 100644
--- /dev/null
+++ b/spyder/utils/tests/data/example.java
@@ -0,0 +1,13 @@
+/******************************************************************************
+ *  Compilation:  javac HelloWorld.java
+ *  Execution:    java HelloWorld
+ ******************************************************************************/
+
+public class HelloWorld {
+
+    public static void main(String[] args) {
+        // Prints "Hello, World" to the terminal window.
+        System.out.println("Hello, World");
+    }
+
+}
diff --git a/spyder/utils/tests/data/example.md b/spyder/utils/tests/data/example.md
new file mode 100644
--- /dev/null
+++ b/spyder/utils/tests/data/example.md
@@ -0,0 +1,41 @@
+*Italic*
+**Bold**
+# Heading 1
+=========	
+## Heading 2
+---------	
+[Link](http://a.com)	
+[Link][1]
+[1]: http://b.org
+![Image](http://url/a.png)
+![Image][1]
+[1]: http://url/b.jpg
+> Blockquote
+A paragraph.
+A paragraph after 1 blank line.
+	
+* List
+* List
+* List
+
+- List
+- List
+- List
+
+1. One
+2. Two
+3. Three
+
+1) One
+2) Two
+3) Three
+
+---	Horizontal Rule
+
+***	Horizontal Rule
+`Inline code` with backticks
+```
+# code block
+print '3 backticks or'
+print 'indent 4 spaces'
+```
diff --git a/spyder/utils/tests/data/example.py b/spyder/utils/tests/data/example.py
new file mode 100644
--- /dev/null
+++ b/spyder/utils/tests/data/example.py
@@ -0,0 +1,63 @@
+"""
+    ********************* VerySimpleWebBrowser ************************
+
+    This is a Very Simple Web Browser implemented over Qt and QtWebKit.
+
+    author: Juan Manuel Garcia <jmg.utn@gmail.com> ~/home
+
+    *******************************************************************
+"""
+
+import numpy as np
+
+print (__file__)
+
+def iterate_1(Z):
+    # Count neighbours
+    N = np.zeros(Z.shape, int)
+    N[1:-1,1:-1] += (Z[0:-2,0:-2] + Z[0:-2,1:-1] + Z[0:-2,2:] +
+                     Z[1:-1,0:-2]                + Z[1:-1,2:] +
+                     Z[2:  ,0:-2] + Z[2:  ,1:-1] + Z[2:  ,2:])
+    N_ = N.ravel()
+    Z_ = Z.ravel()
+
+    # Apply rules
+    R1 = np.argwhere( (Z_==1) & (N_ < 2) )
+    R2 = np.argwhere( (Z_==1) & (N_ > 3) )
+    R3 = np.argwhere( (Z_==1) & ((N_==2) | (N_==3)) )
+    R4 = np.argwhere( (Z_==0) & (N_==3) )
+
+    # Set new values
+    Z_[R1] = 0
+    Z_[R2] = 0
+    Z_[R3] = Z_[R3]
+    Z_[R4] = 1
+
+    # Make sure borders stay null
+    Z[0,:] = Z[-1,:] = Z[:,0] = Z[:,-1] = 0
+
+
+def iterate_2(Z):
+    # Count neighbours
+    N = (Z[0:-2,0:-2] + Z[0:-2,1:-1] + Z[0:-2,2:] +
+         Z[1:-1,0:-2]                + Z[1:-1,2:] +
+         Z[2:  ,0:-2] + Z[2:  ,1:-1] + Z[2:  ,2:])
+
+    # Apply rules
+    birth = (N==3) & (Z[1:-1,1:-1]==0)
+    survive = ((N==2) | (N==3)) & (Z[1:-1,1:-1]==1)
+    Z[...] = 0
+    Z[1:-1,1:-1][birth | survive] = 1
+    return Z
+
+Z = np.array([[0,0,0,0,0,0],
+              [0,0,0,1,0,0],
+              [0,1,0,1,0,0],
+              [0,0,1,1,0,0],
+              [0,0,0,0,0,0],
+              [0,0,0,0,0,0]])
+
+#print(Z)
+for i in range(4):
+    iterate_2(Z)
+    #print (Z)
\ No newline at end of file
diff --git a/spyder/utils/tests/test_get_words.py b/spyder/utils/tests/test_get_words.py
new file mode 100644
--- /dev/null
+++ b/spyder/utils/tests/test_get_words.py
@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © Spyder Project Contributors
+# Licensed under the terms of the MIT License
+
+"""Tests for programs.py."""
+
+from os.path import dirname, abspath, splitext, join
+from spyder.utils.introspection.utils import get_words
+
+HERE = dirname(abspath(__file__))
+TEST_DATA_PATH = join(HERE, 'data')
+
+
+# --- Fixtures
+# -----------------------------------------------------------------------------
+def get_words_by_content(filename):
+    """Test get_words from content in filename."""
+    f_in = join(TEST_DATA_PATH, filename)
+    ext = splitext(f_in)[1]
+    with open(f_in, 'r') as infile:
+        content = infile.read()
+    return  get_words(content=content, extension=ext)
+
+def get_words_by_filename(filename):
+    """Test get_words from  filepath."""
+    filepath = join(TEST_DATA_PATH, filename)
+    return get_words(filepath)
+
+
+# --- Tests
+# -----------------------------------------------------------------------------
+
+def test_get_words_html():
+    """Test for get word from html file syntax."""
+    expected_words = ['DOCTYPE', 'Hello', 'Jamie', 'World', 'body', 'charset', 'en', 'h',
+                      'head', 'here', 'html', 'lang', 'meta', 'p', 'title', 'utf', 'was']
+    assert sorted(expected_words) == sorted(get_words_by_filename("example.html"))
+    assert sorted(expected_words) == sorted(get_words_by_content("example.html"))
+
+def test_get_words_r():
+    """Test for get word from R file syntax."""
+    expected_words = ['Hello', 'function', 'hello', 'name', 's', 'sprintf']
+    assert sorted(expected_words) == sorted(get_words_by_filename("example.R"))
+    assert sorted(expected_words) == sorted(get_words_by_content("example.R"))
+
+def test_get_words_css():
+    """Test for get word from css file syntax."""
+    expected_words = ['DeepSkyBlue', 'nombre-valido', 'text', 'css',
+    		'h', 'color', 'Hello', 'world', 'type', 'style']
+    assert sorted(expected_words) == sorted(get_words_by_filename("example.css"))
+    assert sorted(expected_words) == sorted(get_words_by_content("example.css"))
+
+def test_get_words_python():
+    """Test for get word from html file syntax."""
+    expected_words = ['Apply', 'Browser', 'Count', 'Garcia', 'Juan', 'Make',
+                      'Manuel', 'N', 'N_', 'Qt', 'QtWebKit', 'R', 'Set', 'Simple',
+                      'This', 'Very', 'VerySimpleWebBrowser', 'Web', 'Z', 'Z_', '__file__',
+                      'a', 'and', 'argwhere', 'array', 'as', 'author', 'birth', 'borders',
+                      'com', 'def', 'for', 'gmail', 'home', 'i', 'implemented', 'import',
+                      'in', 'int', 'is', 'iterate_', 'jmg', 'neighbours', 'new', 'np', 'null',
+                      'numpy', 'over', 'print', 'range', 'ravel', 'return', 'rules', 'shape',
+                      'stay', 'sure', 'survive', 'utn', 'values', 'zeros']
+    assert sorted(expected_words) == sorted(get_words_by_filename("example.py"))
+    assert sorted(expected_words) == sorted(get_words_by_content("example.py"))
+
+def test_get_words_java():
+    """Test for get word from java file syntax."""
+    expected_words = ['Compilation', 'Execution', 'Hello', 'HelloWorld', 'Prints', 'String',
+                      'System', 'World', 'args', 'class', 'java', 'javac', 'main', 'out',
+                      'println', 'public', 'static', 'terminal', 'the', 'to', 'void', 'window']
+    assert sorted(expected_words) == sorted(get_words_by_filename("example.java"))
+    assert sorted(expected_words) == sorted(get_words_by_content("example.java"))
+
+def test_get_words_cplusplus():
+    """Test for get word from C++ file syntax."""
+    expected_words = ['Consider', 'Create', 'Implement', 'Obj', 'ObjContainer',
+                      'Postfix', 'Prefix', 'Return', 'SmartPointer', 'Static',
+                      'Zero', 'a', 'above', 'access', 'actual', 'add', 'an', 'back',
+                      'bool', 'call', 'class', 'const', 'container', 'cout', 'definitions',
+                      'do', 'end', 'endl', 'f', 'false', 'for', 'friend', 'g', 'i', 'if',
+                      'implement', 'include', 'index', 'indicates', 'int', 'iostream', 'iterator',
+                      'j', 'list', 'main', 'member', 'method', 'namespace', 'o', 'obj', 'objc',
+                      'oc', 'of', 'operator', 'overload', 'pointer', 'public', 'push', 'return',
+                      's', 'size', 'smart', 'sp', 'standard', 'static', 'std', 'sz', 'the',
+                      'to', 'true', 'using', 'value', 'vector', 'version', 'void', 'while']
+    assert sorted(expected_words) == sorted(get_words_by_filename("example.cpp"))
+    assert sorted(expected_words) == sorted(get_words_by_content("example.cpp"))
+
+def test_get_words_markdown():
+    """Test for get word from markdown file syntax."""
+    expected_words = ['A', 'Blockquote', 'Bold', 'Heading', 'Horizontal', 'Image', 'Inline',
+                      'Italic', 'Link', 'List', 'One', 'Rule', 'Three', 'Two', 'a', 'after',
+                      'b', 'backticks', 'blank', 'block', 'code', 'com', 'http', 'indent',
+                      'jpg', 'line', 'or', 'org', 'paragraph', 'png', 'print', 'spaces',
+                      'url', 'with']
+    assert sorted(expected_words) == sorted(get_words_by_filename("example.md"))
+    assert sorted(expected_words) == sorted(get_words_by_content("example.md"))
+
+def test_get_words_c():
+    """Test for get word from C file syntax."""
+    expected_words = ['f', 'float', 'foo', 'h', 'i', 'include', 'int', 'main',
+                      'n', 'printf', 'pvar', 'return', 'stdio', 'struct',
+                      'var', 'x', 'y']
+    assert sorted(expected_words) == sorted(get_words_by_filename("example.c"))
+    assert sorted(expected_words) == sorted(get_words_by_content("example.c"))