Repository: dart-lang/pub-dev (dart)

Issue:
Test profile import needs to support old archive files that otherwise are rejected
Currently the following test profile will fail to import, as `retry 3.1.0`'s archive have mode bits in the tar headers that are not `644` (rather IIRC `640`). This is an archive check that was introduced after `retry 3.1.0` was published.
```
packages:
  - name: retry
    versions: ['3.1.0', '3.1.1']
```

We should probably have an extra flag to support sidestepping all or selectively ignoring some of the checks we do at upload time, otherwise we can't easily import such old archive versions into a test profile.

Pull request description:
- Fixes #8127 and allows importing `retry 3.1.0`.
- I think this should be enough to import some old archives that are otherwise good quality. If the need arises, we may expand on this feature.

Patch:
diff --git a/app/lib/shared/utils.dart b/app/lib/shared/utils.dart
index e98d013c21..20f64f62f7 100644
--- a/app/lib/shared/utils.dart
+++ b/app/lib/shared/utils.dart
@@ -9,6 +9,7 @@ import 'dart:collection';
 import 'dart:convert';
 import 'dart:io';
 import 'dart:math';
+import 'dart:typed_data';
 
 import 'package:appengine/appengine.dart';
 import 'package:intl/intl.dart';
@@ -300,3 +301,14 @@ extension StringExt on String {
     return v.isEmpty ? null : v;
   }
 }
+
+extension ByteFolderExt on Stream<List<int>> {
+  Future<Uint8List> foldBytes() async {
+    final contents = await toList();
+    final buffer = BytesBuilder(copy: false);
+    for (final chunk in contents) {
+      buffer.add(chunk);
+    }
+    return buffer.toBytes();
+  }
+}


Test patch:
diff --git a/app/lib/tool/test_profile/importer.dart b/app/lib/tool/test_profile/importer.dart
index 01e49b5c71..b8da2454e9 100644
--- a/app/lib/tool/test_profile/importer.dart
+++ b/app/lib/tool/test_profile/importer.dart
@@ -4,16 +4,20 @@
 
 // ignore_for_file: invalid_use_of_visible_for_testing_member
 
+import 'dart:io';
+
 import 'package:_pub_shared/data/admin_api.dart';
 import 'package:_pub_shared/data/package_api.dart';
 import 'package:_pub_shared/search/tags.dart';
 import 'package:meta/meta.dart';
-import 'package:pub_dev/account/auth_provider.dart';
-import 'package:pub_dev/fake/backend/fake_auth_provider.dart';
-import 'package:pub_dev/frontend/handlers/pubapi.client.dart';
-import 'package:pub_dev/service/async_queue/async_queue.dart';
-import 'package:pub_dev/shared/configuration.dart';
-
+import 'package:tar/tar.dart';
+
+import '../../account/auth_provider.dart';
+import '../../fake/backend/fake_auth_provider.dart';
+import '../../frontend/handlers/pubapi.client.dart';
+import '../../service/async_queue/async_queue.dart';
+import '../../shared/configuration.dart';
+import '../../shared/utils.dart';
 import '../utils/pub_api_client.dart';
 import 'import_source.dart';
 import 'models.dart';
@@ -86,8 +90,9 @@ Future<void> importProfile({
           uploaderEmails[rv.version.hashCode.abs() % uploaderEmails.length];
       lastActiveUploaderEmails[rv.package] = uploaderEmail;
 
-      final bytes = pendingBytes['${rv.package}/${rv.version}'] ??
+      var bytes = pendingBytes['${rv.package}/${rv.version}'] ??
           await source.getArchiveBytes(rv.package, rv.version);
+      bytes = await _mayCleanupTarModeBits(bytes);
       try {
         await withHttpPubApiClient(
           bearerToken: createFakeAuthTokenForEmail(uploaderEmail,
@@ -216,3 +221,24 @@ List<String> _potentialActiveEmails(TestProfile profile, String packageName) {
       .members;
   return members.map((m) => m.email).toList();
 }
+
+/// Old archives may contain mode bits that are not supported with the current
+/// upload checks. This method reads the archive and checks for the mode bits.
+/// When the archive bits are not supported, it returns a new archive with the
+/// bits corrected.
+Future<List<int>> _mayCleanupTarModeBits(List<int> bytes) async {
+  final archiveBuilder = ArchiveBuilder();
+  final tarReader =
+      TarReader(Stream.fromIterable([bytes]).transform(gzip.decoder));
+  var needsUpdate = false;
+  while (await tarReader.moveNext()) {
+    final current = tarReader.current;
+    if (current.header.mode != 420) {
+      // 644₈
+      needsUpdate = true;
+    }
+    archiveBuilder.addFileBytes(
+        current.name, await current.contents.foldBytes());
+  }
+  return needsUpdate ? archiveBuilder.toTarGzBytes() : bytes;
+}
diff --git a/app/test/package/screenshots_test.dart b/app/test/package/screenshots_test.dart
index 41d080c234..e3cf2239b7 100644
--- a/app/test/package/screenshots_test.dart
+++ b/app/test/package/screenshots_test.dart
@@ -3,6 +3,7 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'package:pub_dev/package/screenshots/backend.dart';
+import 'package:pub_dev/shared/utils.dart';
 import 'package:test/test.dart';
 
 import '../shared/test_services.dart';
@@ -20,9 +21,7 @@ void main() {
           1);
 
       expect(
-          await imageStorage.bucket
-              .read('new_pkg/1.2.3/image.svg')
-              .fold<List<int>>(<int>[], (buffer, data) => buffer..addAll(data)),
+          await imageStorage.bucket.read('new_pkg/1.2.3/image.svg').foldBytes(),
           [1]);
     });
 
diff --git a/app/test/tool/test_profile/importer_test.dart b/app/test/tool/test_profile/importer_test.dart
index a0dd156e39..f46f9e9d81 100644
--- a/app/test/tool/test_profile/importer_test.dart
+++ b/app/test/tool/test_profile/importer_test.dart
@@ -23,7 +23,7 @@ void main() {
         packages: [
           TestPackage(
             name: 'retry',
-            versions: [TestVersion(version: '3.1.1')],
+            versions: [TestVersion(version: '3.1.0')],
             publisher: 'example.com',
           )
         ],
@@ -37,10 +37,10 @@ void main() {
         final packages = await dbService.query<Package>().run().toList();
         expect(packages.single.name, 'retry');
         expect(packages.single.publisherId, 'example.com');
-        expect(packages.single.latestVersion, '3.1.1');
+        expect(packages.single.latestVersion, '3.1.0');
 
         final versions = await dbService.query<PackageVersion>().run().toList();
-        expect(versions.single.version, '3.1.1');
+        expect(versions.single.version, '3.1.0');
         expect(versions.single.uploader, users.single.userId);
 
         final publishers = await dbService.query<Publisher>().run().toList();