Repository: datanymizer/datanymizer (language: rust)

Issue:
Problem with copying generated columns
Anonymizer throws error when dealing with table with generated columns

Error: db error: ERROR: column "tsv" is a generated column
DETAIL: Generated columns cannot be used in COPY.

pg_dump on its own works fine.

postgresql and pg_dump version is 12.12
pg_datanymizer version is 0.6.0


Discussion / hints:
@ruslan-kurbanov-jr Thanks, It's interesting problem... We'll research it as soon as possible.
The main approach we use is to replace COPY stage in the dump... We'll see how pg_dumper uses it when working with similar columns.
i just ran in to this problem as well. Is there a solution?

pg_dump (PostgreSQL) 16.0
pg_datanymizer version is 0.6.0

A workaround approach for this could be to create a view of the table and dump that instead (using table filters). Do a find and replace on the dump file before importing to rename the view to the table name.

It seems like PG is supposed to work if the dump value for the generated column is `default`: https://stackoverflow.com/questions/64600614/restoring-pg-database-from-dump-fails-due-to-generated-columns
Tried adding the `--inserts` pg_dump argument but it still used COPY. I was hoping that since the error specifically mentioned COPY, then using `--inserts` would allow it to work.

Fix patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3b2c08c..b81ee50 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 ### 🚀 Added
+- Ignore computed columns (previously they would cause an error). [#202](https://github.com/datanymizer/datanymizer/pull/202)
+  ([@gregwebs](https://github.com/gregwebs))
 - Add the JSON transformer [#134](https://github.com/datanymizer/datanymizer/pull/134)
   ([@evgeniy-r](https://github.com/evgeniy-r))
 
diff --git a/datanymizer_dumper/src/postgres/column.rs b/datanymizer_dumper/src/postgres/column.rs
index 365b893..5648896 100644
--- a/datanymizer_dumper/src/postgres/column.rs
+++ b/datanymizer_dumper/src/postgres/column.rs
@@ -68,27 +68,20 @@ mod tests {
     use super::*;
     use std::cmp::Ordering;
 
-    #[test]
-    fn ordering_test() {
-        let col1 = &PgColumn {
-            position: 1,
-            name: String::from("Column1"),
+    fn new_column(position: i32, name: &str) -> PgColumn {
+        PgColumn {
+            position,
+            name: String::from(name),
             data_type: String::new(),
             inner_type: Some(0),
-        };
-        let col2 = &PgColumn {
-            position: 2,
-            name: String::from("Column2"),
-            data_type: String::new(),
-            inner_type: Some(0),
-        };
+        }
+    }
 
-        let col3 = &PgColumn {
-            position: 1,
-            name: String::from("Column1"),
-            data_type: String::new(),
-            inner_type: Some(0),
-        };
+    #[test]
+    fn ordering_test() {
+        let col1 = &new_column(1, "Column1");
+        let col2 = &new_column(2, "Column2");
+        let col3 = &new_column(1, "Column1");
 
         assert_eq!(col1, col3);
         assert_eq!(col1.cmp(col2), Ordering::Less);
diff --git a/datanymizer_dumper/src/postgres/row.rs b/datanymizer_dumper/src/postgres/row.rs
index 950c5d7..ca42a7e 100644
--- a/datanymizer_dumper/src/postgres/row.rs
+++ b/datanymizer_dumper/src/postgres/row.rs
@@ -51,6 +51,15 @@ mod tests {
     use crate::postgres::{column::PgColumn, table::PgTable};
     use datanymizer_engine::Settings;
 
+    fn new_column(position: i32, name: &str) -> PgColumn {
+        PgColumn {
+            position,
+            name: String::from(name),
+            data_type: String::new(),
+            inner_type: Some(0),
+        }
+    }
+
     #[test]
     fn transform() {
         let config = r#"
@@ -74,30 +83,10 @@ mod tests {
 
         let mut table = PgTable::new("table_name".to_string(), "public".to_string());
 
-        let col1 = PgColumn {
-            position: 1,
-            name: String::from("first_name"),
-            data_type: String::new(),
-            inner_type: Some(0),
-        };
-        let col2 = PgColumn {
-            position: 2,
-            name: String::from("middle_name"),
-            data_type: String::new(),
-            inner_type: Some(0),
-        };
-        let col3 = PgColumn {
-            position: 3,
-            name: String::from("last_name"),
-            data_type: String::new(),
-            inner_type: Some(0),
-        };
-        let col4 = PgColumn {
-            position: 4,
-            name: String::from("comment"),
-            data_type: String::new(),
-            inner_type: Some(0),
-        };
+        let col1 = new_column(1, "first_name");
+        let col2 = new_column(2, "middle_name");
+        let col3 = new_column(3, "last_name");
+        let col4 = new_column(4, "comment");
 
         table.set_columns(vec![col1, col2, col3, col4]);
         let row = PgRow::from_string_row("first\tmiddle\tlast\t".to_string(), table);
diff --git a/datanymizer_dumper/src/postgres/schema_inspector.rs b/datanymizer_dumper/src/postgres/schema_inspector.rs
index b9a8f63..b6bc278 100644
--- a/datanymizer_dumper/src/postgres/schema_inspector.rs
+++ b/datanymizer_dumper/src/postgres/schema_inspector.rs
@@ -29,7 +29,7 @@ const TABLE_COLUMNS_QUERY: &str = "SELECT cc.column_name, cc.ordinal_position, c
     ON cc.udt_schema = pn.nspname
     JOIN pg_catalog.pg_type as pt
     ON cc.udt_name = pt.typname AND pn.oid = pt.typnamespace
-    WHERE cc.table_schema = $1 and cc.table_name = $2
+    WHERE cc.table_schema = $1 and cc.table_name = $2 and cc.is_generated = 'NEVER'
     ORDER BY cc.ordinal_position ASC";
 
 const TABLE_SIZE_QUERY: &str =
diff --git a/datanymizer_dumper/src/postgres/table.rs b/datanymizer_dumper/src/postgres/table.rs
index a8a6a8a..36cc7fc 100644
--- a/datanymizer_dumper/src/postgres/table.rs
+++ b/datanymizer_dumper/src/postgres/table.rs
@@ -249,6 +249,15 @@ mod tests {
     use super::*;
     use crate::{postgres::column::PgColumn, Table};
 
+    fn new_column(position: i32, name: &str) -> PgColumn {
+        PgColumn {
+            position,
+            name: String::from(name),
+            data_type: String::new(),
+            inner_type: Some(0),
+        }
+    }
+
     #[test]
     fn table_full_name() {
         let table = PgTable::new(String::from("name"), String::from("public"));
@@ -278,26 +287,9 @@ mod tests {
     fn set_columns() {
         let mut table = PgTable::new(String::from("name"), String::from("public"));
 
-        let col1 = PgColumn {
-            position: 1,
-            name: String::from("col1"),
-            data_type: String::new(),
-            inner_type: Some(0),
-        };
-        let col2 = PgColumn {
-            position: 2,
-            name: String::from("col2"),
-            data_type: String::new(),
-            inner_type: Some(0),
-        };
-        let col3 = PgColumn {
-            // Column positions in Postgres are not always in sequence
-            // (e.g., when we have dropped some column).
-            position: 4,
-            name: String::from("col4"),
-            data_type: String::new(),
-            inner_type: Some(0),
-        };
+        let col1 = new_column(1, "col1");
+        let col2 = new_column(2, "col2");
+        let col3 = new_column(4, "col4");
 
         table.set_columns(vec![col1.clone(), col2.clone(), col3.clone()]);
 
@@ -317,18 +309,8 @@ mod tests {
         }
 
         fn columns() -> Vec<PgColumn> {
-            let col1 = PgColumn {
-                position: 1,
-                name: String::from("col1"),
-                data_type: String::new(),
-                inner_type: Some(0),
-            };
-            let col2 = PgColumn {
-                position: 2,
-                name: String::from("col2"),
-                data_type: String::new(),
-                inner_type: Some(0),
-            };
+            let col1 = new_column(1, "col1");
+            let col2 = new_column(2, "col2");
             vec![col1, col2]
         }
 


Test patch: