internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: intel/dffml

Issue:
feature: add a load function in high level
Add a load function in `high_level.py` so the data from sources can be loaded.
After completing this, #555 can be worked on.


Discussion:

Resolving patch:
diff --git a/dffml/high_level.py b/dffml/high_level.py
--- a/dffml/high_level.py
+++ b/dffml/high_level.py
@@ -3,7 +3,7 @@
 in a lot of quick and dirty python files.
 """
 import pathlib
-from typing import Union, Dict, Any
+from typing import Union, Dict, Any, AsyncIterator
 
 from .record import Record
 from .source.source import Sources, BaseSource
@@ -258,3 +258,60 @@ async def predict(
                 )
                 if update:
                     await sctx.update(record)
+
+
+async def load(source: BaseSource, *args: str) -> AsyncIterator[Record]:
+    """
+    Yields records from a source.
+
+    Yields all the records from the source, if record keys are given then only
+    those records are yielded.
+
+    Parameters
+    ----------
+    source : BaseSource
+        Data source to use. See :doc:`/plugins/dffml_source` for sources and
+        options.
+    *args : str
+        Records to be returned. If empty, all the records in a source will be returned.
+
+    Returns
+    -------
+    asynciterator
+        :py:class:`Record <dffml.record.Record>` object
+
+    Examples
+    --------
+
+    >>> source = CSVSource(filename="save.csv", allowempty=True, readwrite=True)
+    >>>
+    >>> async def main():
+    ...     await save(
+    ...         source,
+    ...         Record("1", data={"features": {"A": 0, "B": 1}}),
+    ...         Record("2", data={"features": {"A": 3, "B": 4}}),
+    ...     )
+    ...
+    ...     # All records in source
+    ...     async for record in load(source):
+    ...         print(record.export())
+    ...
+    ...     # For specific records in a source
+    ...     async for record in load(source, "1"):
+    ...         print(record.export())
+    >>>
+    >>> asyncio.run(main())
+    {'key': '1', 'features': {'A': 0, 'B': 1}, 'extra': {}}
+    {'key': '2', 'features': {'A': 3, 'B': 4}, 'extra': {}}
+    {'key': '1', 'features': {'A': 0, 'B': 1}, 'extra': {}}
+    """
+    async with source:
+        async with source() as sctx:
+            if args:
+                # If specific records are to be loaded
+                for record in args:
+                    yield await sctx.record(record)
+            else:
+                # All the records are loaded
+                async for record in sctx.records():
+                    yield record


Test patch:
Repository: intel/dffml

Issue:
docs: operations: model_predict example usage
We need a doctestable example for the `model_predict`/`dffml.model.predict` operation.

References: https://intel.github.io/dffml/master/plugins/dffml_operation.html#dffml-model-predict


Discussion:

Resolving patch:
diff --git a/dffml/operation/model.py b/dffml/operation/model.py
--- a/dffml/operation/model.py
+++ b/dffml/operation/model.py
@@ -35,6 +35,62 @@ def __post_init__(self):
     ctx_enter={"mctx": (lambda self: self.parent.model())},
 )
 async def model_predict(self, features: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Predict using dffml models.
+
+    Parameters
+    ++++++++++
+    features : dict
+        A dictionary contaning feature name and feature value.
+
+    Returns
+    +++++++
+    dict
+        A dictionary containing prediction.
+
+    Examples
+    ++++++++
+
+    The following example shows how to use model_predict.
+
+    >>> slr_model = SLRModel(
+    ...     features=Features(DefFeature("Years", int, 1)),
+    ...     predict=DefFeature("Salary", int, 1),
+    ... )
+    >>> dataflow = DataFlow(
+    ...     operations={
+    ...         "prediction_using_model": model_predict,
+    ...         "get_single": GetSingle,
+    ...     },
+    ...     configs={"prediction_using_model": ModelPredictConfig(model=slr_model)},
+    ... )
+    >>> dataflow.seed.append(
+    ...     Input(
+    ...         value=[model_predict.op.outputs["prediction"].name],
+    ...         definition=GetSingle.op.inputs["spec"],
+    ...     )
+    ... )
+    >>>
+    >>> async def main():
+    ...     await train(
+    ...         slr_model,
+    ...         {"Years": 0, "Salary": 10},
+    ...         {"Years": 1, "Salary": 20},
+    ...         {"Years": 2, "Salary": 30},
+    ...         {"Years": 3, "Salary": 40},
+    ...     )
+    ...     inputs = [
+    ...        Input(
+    ...            value={"Years": 4}, definition=model_predict.op.inputs["features"],
+    ...        )
+    ...     ]
+    ...     async for ctx, results in MemoryOrchestrator.run(dataflow, inputs):
+    ...         print(results)
+    >>>
+    >>> asyncio.run(main())
+    {'model_predictions': {'Salary': {'confidence': 1.0, 'value': 50.0}}}
+    """
+
     async def records():
         yield Record("", data={"features": features})
 
diff --git a/docs/doctest_header.py b/docs/doctest_header.py
--- a/docs/doctest_header.py
+++ b/docs/doctest_header.py
@@ -25,9 +25,11 @@
 from dffml.df.types import *
 from dffml.util.net import *
 from dffml.df.memory import *
+from dffml.model.slr import *
 from dffml_model_scikit import *
 from dffml.operation.io import *
 from dffml.source.memory import *
+from dffml.operation.model import *
 from dffml.operation.output import *
 from dffml.operation.dataflow import *
 from dffml.operation.preprocess import *


Test patch:
Repository: intel/dffml

Issue:
docs: source: New File source tutorial
The question of how to make a new data source came up [recently](https://gitter.im/dffml/community?at=5e0e4bfbfd580457e7b3e26a). We should have multiple tutorials for sources.

We're going to use https://github.com/intel/dffml/issues/551 as the example for this tutorial, so that issue needs to be closed first.

You'll want to make a

`docs/tutorials/sources/index.rst`

Then take this exist tutorial and move it under

`docs/tutoirials/sources/complex.rst`

You're new one will go under 

`sources/file.rst`

Te idea here is just to explain how you can write a source which reads data from some new filetype, with our example being the `.ini`

Try to follow the new model tutorial format, where we do `literalinclude` and `:lines:` to talk about individual sections of the files we're writing.


Discussion:

Resolving patch:
diff --git a/dffml/source/ini.py b/dffml/source/ini.py
--- a/dffml/source/ini.py
+++ b/dffml/source/ini.py
@@ -23,11 +23,11 @@ class INISource(FileSource, MemorySource):
 
     CONFIG = INISourceConfig
 
-    async def load_fd(self, ifile):
+    async def load_fd(self, fileobj):
         # Creating an instance of configparser
         parser = ConfigParser()
         # Read from a file object
-        parser.read_file(ifile)
+        parser.read_file(fileobj)
         # Get all the sections present in the file
         sections = parser.sections()
 
@@ -46,7 +46,7 @@ async def load_fd(self, ifile):
 
         self.logger.debug("%r loaded %d sections", self, len(self.mem))
 
-    async def dump_fd(self, fd):
+    async def dump_fd(self, fileobj):
         # Create an instance of configparser
         parser = ConfigParser()
 
@@ -61,6 +61,6 @@ async def dump_fd(self, fd):
             parser[section] = section_data
 
         # Write to the fileobject
-        parser.write(fd)
+        parser.write(fileobj)
 
         self.logger.debug("%r saved %d sections", self, len(self.mem))


Test patch:
Repository: pypa/pipenv

Issue:
"pipenv check" checks against the host Python instance rather than the venv's
For example, if I am running `pipenv` under Python 2.7, and have a Pipfile with:
```
[requires]
python_version = "3.5"
```
`pipenv check` will fail even if the project's Python version is correct.


Discussion:
correct, this needs to be fixed. 
fixed!
Woot!
released!
Thanks, amazing turnaround time!
Unfortunately, this appears to be broken on Python 2 (I tested successfully with Python 3): 

```
$ python2 -m pipenv check
Checking PEP 508 requirements...
Traceback (most recent call last):
  File "/usr/lib/python2.7/runpy.py", line 174, in _run_module_as_main
    "__main__", fname, loader, pkg_name)
  File "/usr/lib/python2.7/runpy.py", line 72, in _run_code
    exec code in run_globals
  File "/usr/local/lib/python2.7/dist-packages/pipenv/__main__.py", line 4, in <module>
    cli()
  File "/usr/local/lib/python2.7/dist-packages/click/core.py", line 722, in __call__
    return self.main(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/click/core.py", line 697, in main
    rv = self.invoke(ctx)
  File "/usr/local/lib/python2.7/dist-packages/click/core.py", line 1066, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/usr/local/lib/python2.7/dist-packages/click/core.py", line 895, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/usr/local/lib/python2.7/dist-packages/click/core.py", line 535, in invoke
    return callback(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/pipenv/cli.py", line 552, in check
    results = json.loads(c.out)
  File "/usr/lib/python2.7/json/__init__.py", line 339, in loads
    return _default_decoder.decode(s)
  File "/usr/lib/python2.7/json/decoder.py", line 364, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  File "/usr/lib/python2.7/json/decoder.py", line 382, in raw_decode
    raise ValueError("No JSON object could be decoded")
ValueError: No JSON object could be decoded
```
i will investigate, working fine over here
yes, all is working properly here
On my machine, I get this error in the delegated command:
```
RuntimeError: Bad magic number in .pyc file
```

Resolving patch:
diff --git a/pipenv/cli.py b/pipenv/cli.py
--- a/pipenv/cli.py
+++ b/pipenv/cli.py
@@ -546,7 +546,7 @@ def check():
     click.echo(crayons.yellow('Checking PEP 508 requirements...'))
 
     # Run the PEP 508 checker in the virtualenv.
-    c = delegator.run('{0} {1}'.format(which('python'), pep508checker.__file__))
+    c = delegator.run('{0} {1}'.format(which('python'), pep508checker.__file__.rstrip('cdo')))
     results = json.loads(c.out)
 
     # Load the pipfile.


Test patch:
Repository: pypa/pipenv

Issue:
pexpect.spawn should use list of arguments
`pexpect.spawn` should use list of arguments instead of `str.format`.

```python
import pexpect

path = '/tmp/dir-without-spaces'
child = pexpect.spawn('ls {0}'.format(path))
print(child.read())
child.close()

path = '/tmp/dir with spaces'
child = pexpect.spawn('ls {0}'.format(path))
print(child.read())
child.close()

path = '/tmp/dir with spaces'
child = pexpect.spawn('ls', [path])
print(child.read())
child.close()
```

```
In [1]: run test.py
file1.txt  file2.txt

ls: cannot access '/tmp/dir': No such file or directory
ls: cannot access 'with': No such file or directory
ls: cannot access 'spaces': No such file or directory

file1.txt  file2.txt
```

https://github.com/kennethreitz/pipenv/blob/master/pipenv/cli.py#L626
https://github.com/kennethreitz/pipenv/blob/master/pipenv/cli.py#L667


Discussion:
pull requests accepted!

Resolving patch:
diff --git a/pipenv/cli.py b/pipenv/cli.py
--- a/pipenv/cli.py
+++ b/pipenv/cli.py
@@ -700,9 +700,8 @@ def shell(three=None):
     terminal_dimensions = get_terminal_size()
 
     c = pexpect.spawn(
-            "{0} -i'".format(
-                shell
-            ),
+            shell,
+            ["-i"],
             dimensions=(
                 terminal_dimensions.lines,
                 terminal_dimensions.columns
@@ -739,7 +738,7 @@ def run(command, args, three=None):
 
     # Spawn the new process, and interact with it.
     try:
-        c = pexpect.spawn('{0} {1}'.format(which(command), ' '.join(args)))
+        c = pexpect.spawn(which(command), list(args))
     except pexpect.exceptions.ExceptionPexpect:
         click.echo(crayons.red('The command was not found within the virtualenv!'))
         sys.exit(1)


Test patch:
Repository: pypa/pipenv

Issue:
uninstall --dev



Discussion:
I'm working on this one. 

Resolving patch:
diff --git a/pipenv/cli.py b/pipenv/cli.py
--- a/pipenv/cli.py
+++ b/pipenv/cli.py
@@ -92,6 +92,7 @@ def ensure_project(three=None):
     ensure_pipfile()
     ensure_virtualenv(three=three)
 
+
 def ensure_proper_casing():
     """Ensures proper casing of Pipfile packages, writes to disk."""
     p = project.parsed_pipfile
@@ -642,17 +643,27 @@ def install(package_name=False, more_packages=False, dev=False, three=False, sys
 @click.option('--three/--two', is_flag=True, default=None, help="Use Python 3/2 when creating virtualenv.")
 @click.option('--system', is_flag=True, default=False, help="System pip management.")
 @click.option('--lock', is_flag=True, default=False, help="Lock afterwards.")
-def uninstall(package_name=False, more_packages=False, three=None, system=False, lock=False):
+@click.option('--dev', '-d', is_flag=True, default=False, help="Un-install package(s) from [dev-packages].")
+def uninstall(package_name=False, more_packages=False, three=None, system=False, lock=False, dev=False):
     # Ensure that virtualenv is available.
     ensure_project(three=three)
 
     package_names = (package_name,) + more_packages
+    pipfile_remove = True
 
     # Un-install all dependencies, if none was provided.
     if package_name is False:
-        click.echo(crayons.yellow('No package provided, un-installing all packages.'))
-        do_purge(allow_global=system)
-        sys.exit(1)
+        if not dev:
+            click.echo(crayons.yellow('No package provided, un-installing all packages.'))
+            do_purge(allow_global=system)
+            sys.exit(0)
+        elif 'dev-packages' in project.parsed_pipfile:
+            package_names = project.parsed_pipfile['dev-packages']
+            pipfile_remove = False
+        else:
+            click.echo(crayons.yellow('No dev-packages to uninstall.'))
+            sys.exit(0)
+
 
     for package_name in package_names:
 
@@ -661,8 +672,13 @@ def uninstall(package_name=False, more_packages=False, three=None, system=False,
         c = delegator.run('{0} uninstall {1} -y'.format(which_pip(allow_global=system), package_name))
         click.echo(crayons.blue(c.out))
 
-        click.echo('Removing {0} from Pipfile...'.format(crayons.green(package_name)))
-        project.remove_package_from_pipfile(package_name)
+        if pipfile_remove:
+            if dev:
+                click.echo('Removing {0} from Pipfile\'s {1}...'.format(crayons.green(package_name), crayons.red('[dev-packages]')))
+            else:
+                click.echo('Removing {0} from Pipfile\'s {1}...'.format(crayons.green(package_name), crayons.red('[packages]')))
+
+            project.remove_package_from_pipfile(package_name, dev)
 
         if lock:
             do_lock()


Test patch:
Repository: pypa/pipenv

Issue:
'pep426_name' is not defined when uninstalling
```
$ pipenv install --dev green
Installing green...
...
Adding green to Pipfile's [dev-packages]...
$ pipenv uninstall green 
Un-installing green...
Uninstalling green-2.7.3:
  Successfully uninstalled green-2.7.3

Traceback (most recent call last):
  File "/Users/markus/.local/bin/pipenv", line 11, in <module>
    sys.exit(cli())
  File "/Users/markus/.local/venvs/pipenv/lib/python3.6/site-packages/pipenv/vendor/click/core.py", line 722, in __call__
    return self.main(*args, **kwargs)
  File "/Users/markus/.local/venvs/pipenv/lib/python3.6/site-packages/pipenv/vendor/click/core.py", line 697, in main
    rv = self.invoke(ctx)
  File "/Users/markus/.local/venvs/pipenv/lib/python3.6/site-packages/pipenv/vendor/click/core.py", line 1066, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/Users/markus/.local/venvs/pipenv/lib/python3.6/site-packages/pipenv/vendor/click/core.py", line 895, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/Users/markus/.local/venvs/pipenv/lib/python3.6/site-packages/pipenv/vendor/click/core.py", line 535, in invoke
    return callback(*args, **kwargs)
  File "/Users/markus/.local/venvs/pipenv/lib/python3.6/site-packages/pipenv/cli.py", line 912, in uninstall
    norm_name = pep426_name(package_name)
NameError: name 'pep426_name' is not defined
```

Typo? `pep426_name` -> `pep423_name`


Discussion:

Resolving patch:
diff --git a/pipenv/cli.py b/pipenv/cli.py
--- a/pipenv/cli.py
+++ b/pipenv/cli.py
@@ -909,7 +909,7 @@ def uninstall(package_name=False, more_packages=False, three=None, python=False,
         click.echo(crayons.blue(c.out))
 
         if pipfile_remove:
-            norm_name = pep426_name(package_name)
+            norm_name = pep423_name(package_name)
             if norm_name in project._pipfile['dev-packages'] or norm_name in project._pipfile['packages']:
                 click.echo('Removing {0} from Pipfile...'.format(crayons.green(package_name)))
             else:


Test patch:
Repository: pypa/pipenv

Issue:
Allow file:// uris as pipenv paths
- Any reason not to?
- PR incoming


Discussion:

Resolving patch:
diff --git a/pipenv/utils.py b/pipenv/utils.py
--- a/pipenv/utils.py
+++ b/pipenv/utils.py
@@ -15,7 +15,7 @@
 
 # List of version control systems we support.
 VCS_LIST = ('git', 'svn', 'hg', 'bzr')
-FILE_LIST = ('http://', 'https://', 'ftp://')
+FILE_LIST = ('http://', 'https://', 'ftp://', 'file:///')
 
 requests = requests.session()
 


Test patch:
Repository: pypa/pipenv

Issue:
Pipfile syntax invalid when installing editable requirements with extras_require flag.
For given command:
```bash
pipenv install  "git+https://github.com/requests/requests.git@master#egg=requests[security]"
```
that executes successfully, the produced Pipfile is unfortunately not valid.

```toml
[[source]]
url = "https://pypi.python.org/simple"
verify_ssl = true

[packages."requests[security]"]
git = "https://github.com/requests/requests.git"
ref = "master"
```


Discussion:
I forgot to mention, that I'm willing to provide a fix in exchange for some pointers on how the Pipfile should look like to support `extras_require`.
Thanks for catching this @ticosax. So the easiest fix for this will be copying [this check](https://github.com/kennethreitz/pipenv/blob/master/pipenv/utils.py#L69-L70) into the bottom of the [vcs block](https://github.com/kennethreitz/pipenv/blob/master/pipenv/utils.py#L42-L57). You'll also need to devise a way to cleanup the name so it's `requests` and not `requests[security]`.

Let me know if you have any other questions. Thanks for the interest in helping! 
Thank you @nateprewitt I will give it a try.
First, we will need this: https://github.com/davidfischer/requirements-parser/pull/30

Resolving patch:
diff --git a/pipenv/utils.py b/pipenv/utils.py
--- a/pipenv/utils.py
+++ b/pipenv/utils.py
@@ -162,8 +162,11 @@ def convert_deps_from_pip(dep):
                              'dependencies. Please install remote dependency '
                              'in the form {0}#egg=<package-name>.'.format(req.uri))
 
+        # Extras: e.g. #egg=requests[security]
+        if req.extras:
+            dependency[req.name] = {'extras': req.extras}
         # Crop off the git+, etc part.
-        dependency[req.name] = {req.vcs: req.uri[len(req.vcs) + 1:]}
+        dependency.setdefault(req.name, {}).update({req.vcs: req.uri[len(req.vcs) + 1:]})
 
         # Add --editable, if it's there.
         if req.editable:


Test patch:
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -103,6 +103,13 @@ def test_convert_from_pip(self):
         dep = pipenv.utils.convert_deps_from_pip(dep)
         assert dep == {'MyProject': {'hg': 'http://hg.myproject.org/MyProject', 'ref': 'da39a3ee5e6b'}}
 
+        # vcs dependency with extras_require
+        dep = 'git+https://github.com/requests/requests.git@master#egg=requests[security]'
+        dep = pipenv.utils.convert_deps_from_pip(dep)
+        assert dep == {'requests': {'git': 'https://github.com/requests/requests.git',
+                                    'ref': 'master',
+                                    'extras': ['security']}}
+
         # vcs dependency without #egg
         dep = 'git+https://github.com/kennethreitz/requests.git'
         with pytest.raises(ValueError) as e:
Repository: pypa/pipenv

Issue:
Bug: uninstall dev is not removing package from Pipfile
Doing `pipenv uninstall -d nose` does not remove the package from `Pipfile`.

The cause is because [in here](https://github.com/kennethreitz/pipenv/blob/master/pipenv/cli.py#L1475) happens this `pipfile_remove = False`.
Is there a reason? Removing that line fixes the issue.
I would expect the package to be removed from the Pipfile, whether its a dev or non dev package.
I can submit a PR if required.


Discussion:
pr requested!
this is a bug. 
Ok, I'll write the test and will submit PR
will be waiting :)

Resolving patch:
diff --git a/pipenv/cli.py b/pipenv/cli.py
--- a/pipenv/cli.py
+++ b/pipenv/cli.py
@@ -1472,7 +1472,6 @@ def uninstall(
             )
             package_names = project.parsed_pipfile['dev-packages']
             package_names = package_names.keys()
-            pipfile_remove = False
         else:
             puts(
                 crayons.white('No {0} to uninstall.'.format(


Test patch:
diff --git a/tests/test_pipenv.py b/tests/test_pipenv.py
--- a/tests/test_pipenv.py
+++ b/tests/test_pipenv.py
@@ -167,6 +167,35 @@ def test_pipenv_uninstall(self):
         os.chdir('..')
         delegator.run('rm -fr test_pipenv_uninstall')
 
+    def test_pipenv_uninstall_dev(self):
+        delegator.run('mkdir test_pipenv_uninstall_dev')
+        os.chdir('test_pipenv_uninstall_dev')
+
+        # Build the environment.
+        os.environ['PIPENV_VENV_IN_PROJECT'] = '1'
+        assert delegator.run('touch Pipfile').return_code == 0
+        assert delegator.run('pipenv --python python').return_code == 0
+
+        # Add entries to Pipfile.
+        assert delegator.run('pipenv install pytest --dev').return_code == 0
+
+        pipfile_output = delegator.run('cat Pipfile').out
+        pipfile_list = pipfile_output.split('\n')
+
+        assert '[dev-packages]' in pipfile_list
+        assert 'pytest = "*"' in pipfile_list
+
+        # Uninstall from dev-packages, removing TOML section.
+        assert delegator.run('pipenv uninstall -d pytest').return_code == 0
+
+        pipfile_output = delegator.run('cat Pipfile').out
+        pipfile_list = pipfile_output.split('\n')
+
+        assert 'pytest = "*"' not in pipfile_list
+
+        os.chdir('..')
+        delegator.run('rm -fr test_pipenv_uninstall_dev')
+
     def test_pipenv_run(self):
         working_dir = 'test_pipenv_run'
         delegator.run('mkdir {0}'.format(working_dir))