Repository: doccano/doccano

Issue:
Export data will get nothing, if I choose only approved documents.
if I choose only approved documents, the zip file contains nothing

if I don't choose it, the zip file contains "all.json".

But those txt I have checked.


Discussion:
Would you write your environment? Thank you!
same here, sequence labeling project
I'm new in Doccano and had this issue too.

I found since https://github.com/doccano/doccano/pull/1402/ the "check" button toggle documents as "confirmed", it doesn't approve them anymore. So in database, the approver id remains empty, and there are no document to export. I didn't find any way to approve document anymore, this is sort of disconnected from the frontend.

I'm not sure about the new workflow with document state related in discussions. Do we need another "approve" button in addition to the "complete" button ? Or can it be done in one step ? Or maybe a new export setting to allow confirmed documents exports ?
Is confirmed used for anything aside from filtering?  If so, they should probably just be merged into one status/flag.
i've just run into this issue myself, i worked around with this minor change to sync the approved_by_user with the example state:

```
diff --git a/backend/api/views/example_state.py b/backend/api/views/example_state.py
index 38a82217..57ea018d 100644
--- a/backend/api/views/example_state.py
+++ b/backend/api/views/example_state.py
@@ -1,8 +1,9 @@
 from django.shortcuts import get_object_or_404
 from rest_framework import generics
 from rest_framework.permissions import IsAuthenticated
 
 from ..permissions import IsInProjectOrAdmin
 from ..serializers import ExampleStateSerializer
 
@@ -24,8 +25,42 @@ class ExampleStateList(generics.ListCreateAPIView):
 
     def perform_create(self, serializer):
         queryset = self.get_queryset()
+        example_id = self.kwargs['example_id']
+        example = Example.objects.get(pk=example_id)
+
         if queryset.exists():
+            example.annotations_approved_by = None
+            example.save()
             queryset.delete()
         else:
             example = get_object_or_404(Example, pk=self.kwargs['example_id'])
+            example.annotations_approved_by = self.request.user
+            example.save()
             serializer.save(example=example, confirmed_by=self.request.user)

```
I also have the same problem. I have around 30% of the corpus annotated and checked but if I choose to export only the confirmed documents I only get a 22 byte all.json empty file. Running Doccano as installed by Docker Compose on an Apple M1 chip Mac.
Has there been any progress on fixing this bug?  Or is there a way for us to manually edit the DB to convert "confirmed" to "approved"?
@daleevans Can we use an adapted version of your code to auto-approve all documents marked as confirmed/done by a specific user?

It would also already be helpful to just get a list of all document ids that were marked as confirmed/done by a specific user. Then we could filter the all.json manually after exporting.

Does anyone know how to query the database to get such list?
@mnschmit I ran into this problem and have written some code to extract all ids of examples that were confirmed by a specific user. Take a look:

```
from doccano_api_client import DoccanoClient

client = DoccanoClient(DOCCANO_URL, USERNAME, PASSWORD)

# Get list of all example ids
ids = [example["id"] for example in client.get_examples(4, {"limit": [1200], "offset": [0]})["results"]]

# Query the database for the example states
example_states = [client.get_example_states(4, idx) for idx in ids]

# Filter the states based on if they were confirmed by the desired user and store the approved ids in a list
user_id = 1
confirmed_idx = [state["results"][0]["example"] for state in example_states if state["results"] and state["results"][0]["confirmed_by"] == user_id]
```

`confirmed_idx` then is a list of example ids approved by the specified user

Note: this solution requires the most recent version of [Doccano Client](https://github.com/doccano/doccano-client) (clone the repo and pip install it, the release on pypi is outdated).
@wpnbos Thank you for sharing that approach! I didn't know about the doccano client!

Resolving patch:
diff --git a/backend/data_export/pipeline/repositories.py b/backend/data_export/pipeline/repositories.py
--- a/backend/data_export/pipeline/repositories.py
+++ b/backend/data_export/pipeline/repositories.py
@@ -17,12 +17,15 @@ def __init__(self, project: Project):
     def list(self, export_approved=False) -> Iterator[Record]:
         raise NotImplementedError()
 
+    def create_unlabeled_record(self, example: Example) -> Record:
+        raise NotImplementedError()
+
 
 class FileRepository(BaseRepository):
     def list(self, export_approved=False) -> Iterator[Record]:
         examples = self.project.examples.all()
         if export_approved:
-            examples = examples.exclude(annotations_approved_by=None)
+            examples = examples.exclude(states=None)
 
         for example in examples:
             label_per_user = self.label_per_user(example)
@@ -43,7 +46,10 @@ def list(self, export_approved=False) -> Iterator[Record]:
             # with the user who approved the doc.
             # This means I will allow each user to be able to approve the doc.
             if len(label_per_user) == 0:
-                yield Record(data_id=example.id, data=example.upload_name, label=[], user="unknown", metadata={})
+                yield self.create_unlabeled_record(example)
+
+    def create_unlabeled_record(self, example: Example) -> Record:
+        return Record(data_id=example.id, data=example.upload_name, label=[], user="unknown", metadata=example.meta)
 
     def label_per_user(self, example) -> Dict:
         label_per_user = defaultdict(list)
@@ -72,7 +78,7 @@ def docs(self):
     def list(self, export_approved=False):
         docs = self.docs
         if export_approved:
-            docs = docs.exclude(annotations_approved_by=None)
+            docs = docs.exclude(states=None)
 
         for doc in docs:
             label_per_user = self.label_per_user(doc)
@@ -87,7 +93,10 @@ def list(self, export_approved=False):
             # with the user who approved the doc.
             # This means I will allow each user to be able to approve the doc.
             if len(label_per_user) == 0:
-                yield Record(data_id=doc.id, data=doc.text, label=[], user="unknown", metadata={})
+                yield self.create_unlabeled_record(doc)
+
+    def create_unlabeled_record(self, example: Example) -> Record:
+        return Record(data_id=example.id, data=example.text, label=[], user="unknown", metadata=example.meta)
 
     @abc.abstractmethod
     def label_per_user(self, doc) -> Dict:
@@ -130,6 +139,15 @@ def docs(self):
             "spans__user", "spans__label", "relations__user", "relations__type"
         )
 
+    def create_unlabeled_record(self, example: Example) -> Record:
+        return Record(
+            data_id=example.id,
+            data=example.text,
+            label={"entities": [], "relations": []},
+            user="unknown",
+            metadata=example.meta,
+        )
+
     def label_per_user(self, doc) -> Dict:
         relation_per_user: Dict = defaultdict(list)
         span_per_user: Dict = defaultdict(list)
@@ -186,6 +204,15 @@ def docs(self):
             "categories__user", "categories__label", "spans__user", "spans__label"
         )
 
+    def create_unlabeled_record(self, example: Example) -> Record:
+        return Record(
+            data_id=example.id,
+            data=example.text,
+            label={"entities": [], "cats": []},
+            user="unknown",
+            metadata=example.meta,
+        )
+
     def label_per_user(self, doc) -> Dict:
         category_per_user: Dict[str, List[str]] = defaultdict(list)
         span_per_user: Dict[str, List[SpanType]] = defaultdict(list)


Test patch:
diff --git a/backend/data_export/tests/test_repositories.py b/backend/data_export/tests/test_repositories.py
--- a/backend/data_export/tests/test_repositories.py
+++ b/backend/data_export/tests/test_repositories.py
@@ -23,8 +23,8 @@
 
 
 class TestRepository(unittest.TestCase):
-    def assert_records(self, repository, expected):
-        records = list(repository.list())
+    def assert_records(self, repository, expected, confirmed_only=False):
+        records = list(repository.list(export_approved=confirmed_only))
         self.assertEqual(len(records), len(expected))
         for record, expect in zip(records, expected):
             self.assertEqual(record.data, expect["data"])
@@ -34,9 +34,11 @@ def assert_records(self, repository, expected):
 
 class TestTextClassificationRepository(TestRepository):
     def prepare_data(self, project):
-        self.example = mommy.make("Example", project=project.item, text="example")
-        self.category1 = mommy.make("Category", example=self.example, user=project.admin)
-        self.category2 = mommy.make("Category", example=self.example, user=project.annotator)
+        self.confirmed_example = mommy.make("Example", project=project.item, text="confirmed")
+        self.category1 = mommy.make("Category", example=self.confirmed_example, user=project.admin)
+        self.category2 = mommy.make("Category", example=self.confirmed_example, user=project.annotator)
+        mommy.make("ExampleState", example=self.confirmed_example, confirmed_by=project.admin)
+        self.unconfirmed_example = mommy.make("Example", project=project.item, text="unconfirmed")
 
     def test_list(self):
         project = prepare_project(DOCUMENT_CLASSIFICATION)
@@ -44,15 +46,16 @@ def test_list(self):
         self.prepare_data(project)
         expected = [
             {
-                "data": self.example.text,
+                "data": self.confirmed_example.text,
                 "label": [self.category1.label.text],
                 "user": project.admin.username,
             },
             {
-                "data": self.example.text,
+                "data": self.confirmed_example.text,
                 "label": [self.category2.label.text],
                 "user": project.annotator.username,
             },
+            {"data": self.unconfirmed_example.text, "label": [], "user": "unknown"},
         ]
         self.assert_records(repository, expected)
 
@@ -62,19 +65,44 @@ def test_list_on_collaborative_annotation(self):
         self.prepare_data(project)
         expected = [
             {
-                "data": self.example.text,
+                "data": self.confirmed_example.text,
                 "label": [self.category1.label.text, self.category2.label.text],
                 "user": "all",
-            }
+            },
+            {
+                "data": self.unconfirmed_example.text,
+                "label": [],
+                "user": "all",
+            },
         ]
         self.assert_records(repository, expected)
 
+    def test_list_confirmed_example_only(self):
+        project = prepare_project(DOCUMENT_CLASSIFICATION)
+        repository = TextClassificationRepository(project.item)
+        self.prepare_data(project)
+        expected = [
+            {
+                "data": self.confirmed_example.text,
+                "label": [self.category1.label.text],
+                "user": project.admin.username,
+            },
+            {
+                "data": self.confirmed_example.text,
+                "label": [self.category2.label.text],
+                "user": project.annotator.username,
+            },
+        ]
+        self.assert_records(repository, expected, confirmed_only=True)
+
 
 class TestSeq2seqRepository(TestRepository):
     def prepare_data(self, project):
-        self.example = mommy.make("Example", project=project.item, text="example")
-        self.text1 = mommy.make("TextLabel", example=self.example, user=project.admin)
-        self.text2 = mommy.make("TextLabel", example=self.example, user=project.annotator)
+        self.confirmed_example = mommy.make("Example", project=project.item, text="confirmed")
+        self.text1 = mommy.make("TextLabel", example=self.confirmed_example, user=project.admin)
+        self.text2 = mommy.make("TextLabel", example=self.confirmed_example, user=project.annotator)
+        mommy.make("ExampleState", example=self.confirmed_example, confirmed_by=project.admin)
+        self.unconfirmed_example = mommy.make("Example", project=project.item, text="unconfirmed")
 
     def test_list(self):
         project = prepare_project(SEQ2SEQ)
@@ -82,15 +110,16 @@ def test_list(self):
         self.prepare_data(project)
         expected = [
             {
-                "data": self.example.text,
+                "data": self.confirmed_example.text,
                 "label": [self.text1.text],
                 "user": project.admin.username,
             },
             {
-                "data": self.example.text,
+                "data": self.confirmed_example.text,
                 "label": [self.text2.text],
                 "user": project.annotator.username,
             },
+            {"data": self.unconfirmed_example.text, "label": [], "user": "unknown"},
         ]
         self.assert_records(repository, expected)
 
@@ -100,20 +129,45 @@ def test_list_on_collaborative_annotation(self):
         self.prepare_data(project)
         expected = [
             {
-                "data": self.example.text,
+                "data": self.confirmed_example.text,
                 "label": [self.text1.text, self.text2.text],
                 "user": "all",
-            }
+            },
+            {
+                "data": self.unconfirmed_example.text,
+                "label": [],
+                "user": "all",
+            },
         ]
         self.assert_records(repository, expected)
 
+    def test_list_confirmed_example_only(self):
+        project = prepare_project(SEQ2SEQ)
+        repository = Seq2seqRepository(project.item)
+        self.prepare_data(project)
+        expected = [
+            {
+                "data": self.confirmed_example.text,
+                "label": [self.text1.text],
+                "user": project.admin.username,
+            },
+            {
+                "data": self.confirmed_example.text,
+                "label": [self.text2.text],
+                "user": project.annotator.username,
+            },
+        ]
+        self.assert_records(repository, expected, confirmed_only=True)
+
 
 class TestIntentDetectionSlotFillingRepository(TestRepository):
     def prepare_data(self, project):
-        self.example = mommy.make("Example", project=project.item, text="example")
-        self.category1 = mommy.make("Category", example=self.example, user=project.admin)
-        self.category2 = mommy.make("Category", example=self.example, user=project.annotator)
-        self.span = mommy.make("Span", example=self.example, user=project.admin, start_offset=0, end_offset=1)
+        self.confirmed_example = mommy.make("Example", project=project.item, text="confirmed")
+        self.category1 = mommy.make("Category", example=self.confirmed_example, user=project.admin)
+        self.category2 = mommy.make("Category", example=self.confirmed_example, user=project.annotator)
+        self.span = mommy.make("Span", example=self.confirmed_example, user=project.admin, start_offset=0, end_offset=1)
+        mommy.make("ExampleState", example=self.confirmed_example, confirmed_by=project.admin)
+        self.unconfirmed_example = mommy.make("Example", project=project.item, text="unconfirmed")
 
     def test_list(self):
         project = prepare_project(INTENT_DETECTION_AND_SLOT_FILLING)
@@ -121,7 +175,7 @@ def test_list(self):
         self.prepare_data(project)
         expected = [
             {
-                "data": self.example.text,
+                "data": self.confirmed_example.text,
                 "label": {
                     "cats": [self.category1.label.text],
                     "entities": [(self.span.start_offset, self.span.end_offset, self.span.label.text)],
@@ -129,13 +183,14 @@ def test_list(self):
                 "user": project.admin.username,
             },
             {
-                "data": self.example.text,
+                "data": self.confirmed_example.text,
                 "label": {
                     "cats": [self.category2.label.text],
                     "entities": [],
                 },
                 "user": project.annotator.username,
             },
+            {"data": self.unconfirmed_example.text, "label": {"cats": [], "entities": []}, "user": "unknown"},
         ]
         self.assert_records(repository, expected)
 
@@ -145,22 +200,53 @@ def test_list_on_collaborative_annotation(self):
         self.prepare_data(project)
         expected = [
             {
-                "data": self.example.text,
+                "data": self.confirmed_example.text,
                 "label": {
                     "cats": [self.category1.label.text, self.category2.label.text],
                     "entities": [(self.span.start_offset, self.span.end_offset, self.span.label.text)],
                 },
                 "user": "all",
-            }
+            },
+            {"data": self.unconfirmed_example.text, "label": {"cats": [], "entities": []}, "user": "all"},
         ]
         self.assert_records(repository, expected)
 
+    def test_list_confirmed_example_only(self):
+        project = prepare_project(INTENT_DETECTION_AND_SLOT_FILLING)
+        repository = IntentDetectionSlotFillingRepository(project.item)
+        self.prepare_data(project)
+        expected = [
+            {
+                "data": self.confirmed_example.text,
+                "label": {
+                    "cats": [self.category1.label.text],
+                    "entities": [(self.span.start_offset, self.span.end_offset, self.span.label.text)],
+                },
+                "user": project.admin.username,
+            },
+            {
+                "data": self.confirmed_example.text,
+                "label": {
+                    "cats": [self.category2.label.text],
+                    "entities": [],
+                },
+                "user": project.annotator.username,
+            },
+        ]
+        self.assert_records(repository, expected, confirmed_only=True)
+
 
 class TestSequenceLabelingRepository(TestRepository):
     def prepare_data(self, project):
-        self.example = mommy.make("Example", project=project.item, text="example")
-        self.span1 = mommy.make("Span", example=self.example, user=project.admin, start_offset=0, end_offset=1)
-        self.span2 = mommy.make("Span", example=self.example, user=project.annotator, start_offset=1, end_offset=2)
+        self.confirmed_example = mommy.make("Example", project=project.item, text="confirmed")
+        self.span1 = mommy.make(
+            "Span", example=self.confirmed_example, user=project.admin, start_offset=0, end_offset=1
+        )
+        self.span2 = mommy.make(
+            "Span", example=self.confirmed_example, user=project.annotator, start_offset=1, end_offset=2
+        )
+        mommy.make("ExampleState", example=self.confirmed_example, confirmed_by=project.admin)
+        self.unconfirmed_example = mommy.make("Example", project=project.item, text="unconfirmed")
 
     def test_list(self):
         project = prepare_project(SEQUENCE_LABELING)
@@ -168,15 +254,16 @@ def test_list(self):
         self.prepare_data(project)
         expected = [
             {
-                "data": self.example.text,
+                "data": self.confirmed_example.text,
                 "label": [(self.span1.start_offset, self.span1.end_offset, self.span1.label.text)],
                 "user": project.admin.username,
             },
             {
-                "data": self.example.text,
+                "data": self.confirmed_example.text,
                 "label": [(self.span2.start_offset, self.span2.end_offset, self.span2.label.text)],
                 "user": project.annotator.username,
             },
+            {"data": self.unconfirmed_example.text, "label": [], "user": "unknown"},
         ]
         self.assert_records(repository, expected)
 
@@ -186,28 +273,49 @@ def test_list_on_collaborative_annotation(self):
         self.prepare_data(project)
         expected = [
             {
-                "data": self.example.text,
+                "data": self.confirmed_example.text,
                 "label": [
                     (self.span1.start_offset, self.span1.end_offset, self.span1.label.text),
                     (self.span2.start_offset, self.span2.end_offset, self.span2.label.text),
                 ],
                 "user": "all",
-            }
+            },
+            {"data": self.unconfirmed_example.text, "label": [], "user": "all"},
         ]
         self.assert_records(repository, expected)
 
+    def test_list_confirmed_example_only(self):
+        project = prepare_project(SEQUENCE_LABELING)
+        repository = SequenceLabelingRepository(project.item)
+        self.prepare_data(project)
+        expected = [
+            {
+                "data": self.confirmed_example.text,
+                "label": [(self.span1.start_offset, self.span1.end_offset, self.span1.label.text)],
+                "user": project.admin.username,
+            },
+            {
+                "data": self.confirmed_example.text,
+                "label": [(self.span2.start_offset, self.span2.end_offset, self.span2.label.text)],
+                "user": project.annotator.username,
+            },
+        ]
+        self.assert_records(repository, expected, confirmed_only=True)
+
 
 class TestRelationExtractionRepository(TestRepository):
     def test_list(self):
         project = prepare_project(SEQUENCE_LABELING, use_relation=True)
-        example = mommy.make("Example", project=project.item, text="example")
-        span1 = mommy.make("Span", example=example, user=project.admin, start_offset=0, end_offset=1)
-        span2 = mommy.make("Span", example=example, user=project.admin, start_offset=1, end_offset=2)
-        relation = mommy.make("Relation", from_id=span1, to_id=span2, example=example, user=project.admin)
+        confirmed_example = mommy.make("Example", project=project.item, text="example")
+        span1 = mommy.make("Span", example=confirmed_example, user=project.admin, start_offset=0, end_offset=1)
+        span2 = mommy.make("Span", example=confirmed_example, user=project.admin, start_offset=1, end_offset=2)
+        relation = mommy.make("Relation", from_id=span1, to_id=span2, example=confirmed_example, user=project.admin)
+        mommy.make("ExampleState", example=confirmed_example, confirmed_by=project.admin)
+        unconfirmed_example = mommy.make("Example", project=project.item, text="unconfirmed")
         repository = RelationExtractionRepository(project.item)
         expected = [
             {
-                "data": example.text,
+                "data": confirmed_example.text,
                 "label": {
                     "entities": [
                         {
@@ -228,7 +336,8 @@ def test_list(self):
                     ],
                 },
                 "user": project.admin.username,
-            }
+            },
+            {"data": unconfirmed_example.text, "label": {"entities": [], "relations": []}, "user": "unknown"},
         ]
         self.assert_records(repository, expected)
 
@@ -266,12 +375,50 @@ def test_list_on_collaborative_annotation(self):
         ]
         self.assert_records(repository, expected)
 
+    def test_list_confirmed_example_only(self):
+        project = prepare_project(SEQUENCE_LABELING, use_relation=True)
+        confirmed_example = mommy.make("Example", project=project.item, text="example")
+        span1 = mommy.make("Span", example=confirmed_example, user=project.admin, start_offset=0, end_offset=1)
+        span2 = mommy.make("Span", example=confirmed_example, user=project.admin, start_offset=1, end_offset=2)
+        relation = mommy.make("Relation", from_id=span1, to_id=span2, example=confirmed_example, user=project.admin)
+        mommy.make("ExampleState", example=confirmed_example, confirmed_by=project.admin)
+        mommy.make("Example", project=project.item, text="unconfirmed")
+        repository = RelationExtractionRepository(project.item)
+        expected = [
+            {
+                "data": confirmed_example.text,
+                "label": {
+                    "entities": [
+                        {
+                            "id": span1.id,
+                            "start_offset": span1.start_offset,
+                            "end_offset": span1.end_offset,
+                            "label": span1.label.text,
+                        },
+                        {
+                            "id": span2.id,
+                            "start_offset": span2.start_offset,
+                            "end_offset": span2.end_offset,
+                            "label": span2.label.text,
+                        },
+                    ],
+                    "relations": [
+                        {"id": relation.id, "from_id": span1.id, "to_id": span2.id, "type": relation.type.text}
+                    ],
+                },
+                "user": project.admin.username,
+            },
+        ]
+        self.assert_records(repository, expected, confirmed_only=True)
+
 
 class TestSpeech2TextRepository(TestRepository):
     def prepare_data(self, project):
-        self.example = mommy.make("Example", project=project.item, text="example")
-        self.text1 = mommy.make("TextLabel", example=self.example, user=project.admin)
-        self.text2 = mommy.make("TextLabel", example=self.example, user=project.annotator)
+        self.confirmed_example = mommy.make("Example", project=project.item)
+        self.text1 = mommy.make("TextLabel", example=self.confirmed_example, user=project.admin)
+        self.text2 = mommy.make("TextLabel", example=self.confirmed_example, user=project.annotator)
+        mommy.make("ExampleState", example=self.confirmed_example, confirmed_by=project.admin)
+        self.unconfirmed_example = mommy.make("Example", project=project.item, text="unconfirmed")
 
     def test_list(self):
         project = prepare_project(SPEECH2TEXT)
@@ -279,15 +426,16 @@ def test_list(self):
         self.prepare_data(project)
         expected = [
             {
-                "data": self.example.upload_name,
+                "data": self.confirmed_example.upload_name,
                 "label": [self.text1.text],
                 "user": project.admin.username,
             },
             {
-                "data": self.example.upload_name,
+                "data": self.confirmed_example.upload_name,
                 "label": [self.text2.text],
                 "user": project.annotator.username,
             },
+            {"data": self.unconfirmed_example.upload_name, "label": [], "user": "unknown"},
         ]
         self.assert_records(repository, expected)
 
@@ -297,19 +445,40 @@ def test_list_on_collaborative_annotation(self):
         self.prepare_data(project)
         expected = [
             {
-                "data": self.example.upload_name,
+                "data": self.confirmed_example.upload_name,
                 "label": [self.text1.text, self.text2.text],
                 "user": "all",
-            }
+            },
+            {"data": self.unconfirmed_example.upload_name, "label": [], "user": "all"},
         ]
         self.assert_records(repository, expected)
 
+    def test_list_confirmed_example_only(self):
+        project = prepare_project(SPEECH2TEXT)
+        repository = Speech2TextRepository(project.item)
+        self.prepare_data(project)
+        expected = [
+            {
+                "data": self.confirmed_example.upload_name,
+                "label": [self.text1.text],
+                "user": project.admin.username,
+            },
+            {
+                "data": self.confirmed_example.upload_name,
+                "label": [self.text2.text],
+                "user": project.annotator.username,
+            },
+        ]
+        self.assert_records(repository, expected, confirmed_only=True)
+
 
 class TestFileRepository(TestRepository):
     def prepare_data(self, project):
-        self.example = mommy.make("Example", project=project.item, text="example")
-        self.category1 = mommy.make("Category", example=self.example, user=project.admin)
-        self.category2 = mommy.make("Category", example=self.example, user=project.annotator)
+        self.confirmed_example = mommy.make("Example", project=project.item, text="example")
+        self.category1 = mommy.make("Category", example=self.confirmed_example, user=project.admin)
+        self.category2 = mommy.make("Category", example=self.confirmed_example, user=project.annotator)
+        mommy.make("ExampleState", example=self.confirmed_example, confirmed_by=project.admin)
+        self.unconfirmed_example = mommy.make("Example", project=project.item, text="unconfirmed")
 
     def test_list(self):
         project = prepare_project(IMAGE_CLASSIFICATION)
@@ -317,15 +486,16 @@ def test_list(self):
         self.prepare_data(project)
         expected = [
             {
-                "data": self.example.upload_name,
+                "data": self.confirmed_example.upload_name,
                 "label": [self.category1.label.text],
                 "user": project.admin.username,
             },
             {
-                "data": self.example.upload_name,
+                "data": self.confirmed_example.upload_name,
                 "label": [self.category2.label.text],
                 "user": project.annotator.username,
             },
+            {"data": self.unconfirmed_example.upload_name, "label": [], "user": "unknown"},
         ]
         self.assert_records(repository, expected)
 
@@ -335,9 +505,28 @@ def test_list_on_collaborative_annotation(self):
         self.prepare_data(project)
         expected = [
             {
-                "data": self.example.upload_name,
+                "data": self.confirmed_example.upload_name,
                 "label": [self.category1.label.text, self.category2.label.text],
                 "user": "all",
-            }
+            },
+            {"data": self.unconfirmed_example.upload_name, "label": [], "user": "all"},
         ]
         self.assert_records(repository, expected)
+
+    def test_list_confirmed_example_only(self):
+        project = prepare_project(IMAGE_CLASSIFICATION)
+        repository = FileRepository(project.item)
+        self.prepare_data(project)
+        expected = [
+            {
+                "data": self.confirmed_example.upload_name,
+                "label": [self.category1.label.text],
+                "user": project.admin.username,
+            },
+            {
+                "data": self.confirmed_example.upload_name,
+                "label": [self.category2.label.text],
+                "user": project.annotator.username,
+            },
+        ]
+        self.assert_records(repository, expected, confirmed_only=True)