Repository: xataio/client-ts
Pull request #151 (2022-05-06 12:36:41)

Importer fixes
- Fix stopping / pausing the CSV importer. The csvtojson library in theory supports promises but was not waiting for the promise I was returning. Changing the callback to `async` to always return a promise and having a flag to stop processing lines when we decide, fixed error with files larger than the batch size.
- Use `null` instead of empty string for emails when an empty string is found. Otherwise the backend returns a 400.



Patch:
diff --git a/.changeset/early-frogs-raise.md b/.changeset/early-frogs-raise.md
new file mode 100644
index 00000000..9bf07599
--- /dev/null
+++ b/.changeset/early-frogs-raise.md
@@ -0,0 +1,5 @@
+---
+'@xata.io/importer': patch
+---
+
+Fix processing files larger than the batch size
diff --git a/importer/src/csv.ts b/importer/src/csv.ts
index 40f2ab54..53e10cff 100644
--- a/importer/src/csv.ts
+++ b/importer/src/csv.ts
@@ -21,25 +21,28 @@ function initConverter({ noheader }: ParseOptions) {
 
 function process(converter: Converter, { callback, batchSize = 100, columns, maxRows }: ParseOptions) {
   let rows = 0;
+  // Even after calling converter.end() it seems that csvtojson reads one or two lines more.
+  // We want to avoid that.
+  let stopped = false;
   return new Promise((resolve, reject) => {
     let lines: string[][] = [];
     converter.on('header', (header) => {
       if (!columns) columns = header;
     });
     converter.subscribe(
-      (line) => {
+      async (line) => {
+        if (stopped) return;
+
         lines.push(line);
         if (lines.length >= batchSize) {
-          const p = callback(lines, columns, rows);
+          const stop = await callback(lines, columns, rows);
+          if (stop) {
+            stopped = true;
+            converter.end();
+            return;
+          }
+
           lines = [];
-          return p
-            .then((stop) => {
-              if (stop) converter.end();
-            })
-            .catch((err) => {
-              converter.end();
-              reject(err);
-            });
         }
         rows++;
         if (maxRows && rows > maxRows) {
@@ -48,6 +51,7 @@ function process(converter: Converter, { callback, batchSize = 100, columns, max
       },
       reject,
       () => {
+        if (stopped) return;
         const p = lines.length > 0 ? callback(lines, columns, rows) : Promise.resolve();
         p.then(resolve).catch(reject);
       }
diff --git a/importer/src/utils.ts b/importer/src/utils.ts
index 536a91eb..b3e25227 100644
--- a/importer/src/utils.ts
+++ b/importer/src/utils.ts
@@ -92,6 +92,8 @@ export function parseRow(values: string[], types: string[]) {
       return ['true', 'false'].includes(val) ? val === 'true' : null;
     } else if (type === 'multiple') {
       return parseArray(val);
+    } else if (type === 'email') {
+      return val || null;
     }
     return val;
   });


Test patch:
diff --git a/importer/src/utils.test.ts b/importer/src/utils.test.ts
index 6d2e465b..68dec72d 100644
--- a/importer/src/utils.test.ts
+++ b/importer/src/utils.test.ts
@@ -147,4 +147,8 @@ describe('parseRow', () => {
   test('parses multiples', () => {
     expect(parseRow(['[]', '[1,2,3]'], ['multiple', 'multiple'])).toEqual([[], ['1', '2', '3']]);
   });
+
+  test('parses emails', () => {
+    expect(parseRow(['', 'foo@example.com'], ['email', 'email'])).toEqual([null, 'foo@example.com']);
+  });
 });