Repository: beetbox/beets

Issue:
Re-imports don't update `data_source` flex attr
### Problem

When importing an album or a singleton that was previously imported already, the `data_source` flexible attribute is not updated. It stays as **initially imported**.


### Setup

* OS: ---
* Python version: ---
* beets version: 1.6.1 dev-latest
* Turning off plugins made problem go away (yes/no): no


### Proposed solution 

`data_source` needs to be excluded in `reimport_metadata()`




Discussion:
Even album re-import does not update `data_source`.
> 

Thanks for testing and clarifying. I was actually assuming that this is also the case but I wasn't sure yet. So, many thanks, that is very helpful! :-)

I'm pretty sure the problem is located here:

https://github.com/beetbox/beets/blob/4194914e94077f673cf920d595580696b4485305/beets/importer.py#L810-L812

On a reimport this function looks up what `data_source` is in the library already and then simply re-applies it. Similar to the ID fields over at PR #4708, `data_source` would need to be excluded from this "preserving mechanism".


We should investigate older issues and the one that implemented the `reimport_metadata()` method and make sure this is not "intentional" behaviour. If it was, I could think about making it configurable. As I see it now though, it feels like a bug, data_source is an informational field and after a reimport it is just wrong. I _do_ use this field sometimes to save that "I tagged this manuall" with `beet mod data_source=MANUAL query`, but also then _if_ I do find an appropriate album in some metadata source, it makes sense that this field gets overwritten and tells the truth.

Resolving patch:
diff --git a/beets/importer.py b/beets/importer.py
--- a/beets/importer.py
+++ b/beets/importer.py
@@ -50,6 +50,11 @@
 SINGLE_ARTIST_THRESH = 0.25
 PROGRESS_KEY = 'tagprogress'
 HISTORY_KEY = 'taghistory'
+# Album and item flexble attrbutes that should not be preserved on reimports.
+REIMPORT_FRESH_FIELDS_ALBUM = ['data_source']
+REIMPORT_FRESH_FIELDS_ITEM = ['data_source', 'bandcamp_album_id',
+                              'spotify_album_id', 'deezer_album_id',
+                              'beatport_album_id']
 
 # Global logger.
 log = logging.getLogger('beets')
@@ -811,21 +816,46 @@ def reimport_metadata(self, lib):
         """For reimports, preserves metadata for reimported items and
         albums.
         """
+        def _reduce_and_log(new_obj, existing_fields, overwrite_keys):
+            """Some flexible attributes should be overwritten (rather than
+            preserved) on reimports; Copies existing_fields, logs and removes
+            entries that should not be preserved and returns a dict containing
+            those fields left to actually be preserved.
+            """
+            noun = 'album' if isinstance(new_obj, library.Album) else 'item'
+            existing_fields = dict(existing_fields)
+            overwritten_fields = [k for k in existing_fields
+                                  if k in overwrite_keys
+                                  and existing_fields.get(k) != new_obj.get(k)]
+            if overwritten_fields:
+                log.debug(
+                    'Reimported {} {}. Not preserving flexible attributes {}. '
+                    'Path: {}',
+                    noun, new_obj.id, overwritten_fields,
+                    displayable_path(new_obj.path))
+                for key in overwritten_fields:
+                    del existing_fields[key]
+            return existing_fields
+
         if self.is_album:
             replaced_album = self.replaced_albums.get(self.album.path)
             if replaced_album:
+                album_fields = _reduce_and_log(self.album,
+                                               replaced_album._values_flex,
+                                               REIMPORT_FRESH_FIELDS_ALBUM)
                 self.album.added = replaced_album.added
-                self.album.update(replaced_album._values_flex)
+                self.album.update(album_fields)
                 self.album.artpath = replaced_album.artpath
                 self.album.store()
                 log.debug(
-                    'Reimported album: added {0}, flexible '
-                    'attributes {1} from album {2} for {3}',
-                    self.album.added,
-                    replaced_album._values_flex.keys(),
-                    replaced_album.id,
-                    displayable_path(self.album.path)
-                )
+                    "Reimported album {}. Preserving attribute ['added']. "
+                    "Path: {}",
+                    self.album.id, displayable_path(self.album.path))
+                log.debug(
+                    'Reimported album {}. Preserving flexible attributes {}. '
+                    'Path: {}',
+                    self.album.id, list(album_fields.keys()),
+                    displayable_path(self.album.path))
 
         for item in self.imported_items():
             dup_items = self.replaced_items[item]
@@ -833,33 +863,17 @@ def reimport_metadata(self, lib):
                 if dup_item.added and dup_item.added != item.added:
                     item.added = dup_item.added
                     log.debug(
-                        'Reimported item added {0} '
-                        'from item {1} for {2}',
-                        item.added,
-                        dup_item.id,
-                        displayable_path(item.path)
-                    )
-                # We exclude certain flexible attributes from the preserving
-                # process since they might have been fetched from MusicBrainz
-                # and been set in beets.autotag.apply_metadata().
-                # discogs_albumid might also have been set but is not a
-                # flexible attribute, thus no exclude is required.
-                if item.get('bandcamp_album_id'):
-                    dup_item.bandcamp_album_id = item.bandcamp_album_id
-                if item.get('spotify_album_id'):
-                    dup_item.spotify_album_id = item.spotify_album_id
-                if item.get('deezer_album_id'):
-                    dup_item.deezer_album_id = item.deezer_album_id
-                if item.get('beatport_album_id'):
-                    dup_item.beatport_album_id = item.beatport_album_id
-                item.update(dup_item._values_flex)
+                        "Reimported item {}. Preserving attribute ['added']. "
+                        "Path: {}",
+                        item.id, displayable_path(item.path))
+                item_fields = _reduce_and_log(item, dup_item._values_flex,
+                                              REIMPORT_FRESH_FIELDS_ITEM)
+                item.update(item_fields)
                 log.debug(
-                    'Reimported item flexible attributes {0} '
-                    'from item {1} for {2}',
-                    dup_item._values_flex.keys(),
-                    dup_item.id,
-                    displayable_path(item.path)
-                )
+                    'Reimported item {}. Preserving flexible attributes {}. '
+                    'Path: {}',
+                    item.id, list(item_fields.keys()),
+                    displayable_path(item.path))
                 item.store()
 
     def remove_replaced(self, lib):


Test patch:
diff --git a/test/test_importer.py b/test/test_importer.py
--- a/test/test_importer.py
+++ b/test/test_importer.py
@@ -139,7 +139,8 @@ def _make_album_match(self, artist, album, tracks, distance=0, missing=0):
             va=False,
             album_id='albumid' + id,
             artist_id='artistid' + id,
-            albumtype='soundtrack'
+            albumtype='soundtrack',
+            data_source='match_source'
         )
 
 
@@ -1723,6 +1724,7 @@ def setUp(self):
         album = self.add_album_fixture()
         album.added = 4242.0
         album.foo = 'bar'  # Some flexible attribute.
+        album.data_source = 'original_source'
         album.store()
         item = album.items().get()
         item.baz = 'qux'
@@ -1804,6 +1806,12 @@ def test_reimported_item_preserves_art(self):
         if new_artpath != old_artpath:
             self.assertNotExists(old_artpath)
 
+    def test_reimported_album_not_preserves_flexattr(self):
+        self._setup_session()
+        self.assertEqual(self._album().data_source, 'original_source')
+        self.importer.run()
+        self.assertEqual(self._album().data_source, 'match_source')
+
 
 class ImportPretendTest(_common.TestCase, ImportHelper):
     """ Test the pretend commandline option