-- DBMS: bigquery
CREATE TABLE `isb-cgc-bq.HTAN_versioned.scATACseq_level4_metadata_r3`
(
  Component STRING OPTIONS(description="Category of metadata (e.g. Diagnosis, Biospecimen, scRNA-seq Level 1, etc.); provide the same one for all items/rows."),
  Filename STRING OPTIONS(description="Name of a file"),
  File_Format STRING OPTIONS(description="Format of a file (e.g. txt, csv, fastq, bam, etc.)"),
  HTAN_Parent_Data_File_ID STRING OPTIONS(description="HTAN Data File Identifier indicating the file(s) from which these files were derived"),
  HTAN_Data_File_ID STRING OPTIONS(description="Self-identifier for this data file - HTAN ID of this file HTAN ID SOP (eg HTANx_yyy_zzz)"),
  scATACseq_Workflow_Type STRING OPTIONS(description="Generic name for the workflow used to analyze a data set."),
  scATACseq_Workflow_Parameters_Description STRING OPTIONS(description="Parameters used to run the scATAC-seq workflow."),
  Workflow_Version STRING OPTIONS(description="Major version of the workflow (e.g. Cell Ranger v3.1)"),
  Workflow_Link STRING OPTIONS(description="Link to workflow or command. DockStore.org recommended. URL"),
  entityId STRING OPTIONS(description="Synapse ID of entity"),
  HTAN_Center STRING OPTIONS(description="Name of HTAN Center e.g. HTAN HTAPP"),
  Data_Release STRING OPTIONS(description="Inclusion in HTAN Data Release e.g. Release 1.0"),
  Manifest_Version INT64 OPTIONS(description="Latest version of metadata manifest"),
  Uuid STRING OPTIONS(description="Unique identifier of entity e.g. a96059eb-81af-4b24-ae50-9242c0d8f819"),
  File_Size INT64 OPTIONS(description="Size of file (bucket object) in bytes"),
  md5 STRING OPTIONS(description="md5 hash of file"),
  CDS_Release STRING OPTIONS(description="CDS transfer package in which file was included e.g. v22.6.1.seq")
)
OPTIONS(
  friendly_name="HTAN SCATACSEQ LEVEL 4 METADATA R3 VERSIONED",
  description="This table contains the file metadata for the fourth level (t-SNE plot coordinates, etc.) single cell ATAC sequencing. Data was extracted from HTAN Synapse projects in April 2023. More details: see https://data.humantumoratlas.org/standard/sequencing",
  labels=[("access", "open"), ("data_type", "file_metadata"), ("program", "htan"), ("source", "htan"), ("status", "current"), ("category", "metadata")]
);

INSERT INTO `isb-cgc-bq.HTAN_versioned.scATACseq_level4_metadata_r3` (Component, Filename, File_Format, HTAN_Parent_Data_File_ID, HTAN_Data_File_ID, scATACseq_Workflow_Type, scATACseq_Workflow_Parameters_Description, Workflow_Version, Workflow_Link, entityId, HTAN_Center, Data_Release, Manifest_Version, Uuid, File_Size, md5, CDS_Release) VALUES ('ScATAC-seqLevel4', 'single_cell_ATACseq_level_4_clean/metadata_scATAC_healthydonor.tsv', 'tsv', 'HTA4_29_50001,HTA4_29_50002,HTA4_29_50003,HTA4_29_50004,HTA4_29_50005,HTA4_29_50006,HTA4_29_50007,HTA4_29_50008,HTA4_29_50009,HTA4_29_50010,HTA4_29_50011,HTA4_29_50012,HTA4_29_50013,HTA4_29_50014,HTA4_29_50015,HTA4_29_50016,HTA4_29_50017,HTA4_29_50018,HTA4_28_50019,HTA4_28_50020,HTA4_28_50021,HTA4_28_50022,HTA4_28_50023,HTA4_28_50024,HTA4_28_50025,HTA4_28_50026,HTA4_28_50027,HTA4_28_50028,HTA4_28_50029,HTA4_28_50030,HTA4_28_50031,HTA4_28_50032,HTA4_28_50033,HTA4_28_50034,HTA4_28_50035,HTA4_28_50036,HTA4_30_50037,HTA4_30_50038,HTA4_30_50039,HTA4_30_50040,HTA4_30_50041,HTA4_30_50042,HTA4_30_50043,HTA4_30_50044,HTA4_30_50045,HTA4_30_50046,HTA4_30_50047,HTA4_30_50048,HTA4_30_50049,HTA4_30_50050,HTA4_30_50051,HTA4_30_50052,HTA4_30_50053,HTA4_30_50054', 'HTA4_0_80001', 'Other', 'Similar to integrating data from 18 patients, we first merged the peaks from all healthy donors using the mergePeaks module in scATAC-pro and reconstructed the peak-by-cell count matrix using combined peaks and the reConstMtx module in scATAC-pro. The 21 (7 healthy donors each with 3 sorted samples) peak-by-cell count matrices were then combined into a bigger peak-by-cell count matrix and converted to a Seurat object. We next applied the same procedure as integrating patient scATAC-Seq data, except using the first 30 PCs.', 'Other', 'https://doi.org/10.1182/blood.2021013442', 'syn51301056', 'HTAN CHOP', 'Release 3.0', 2, '351796ae-36c9-4806-9a77-ab1e4af389ab', 2537112, '66d09b8cc3808bfd8f84ce0a463e2e47', NULL), ('ScATAC-seqLevel4', 'single_cell_ATACseq_level_4_clean/seurat_9HDsamplesPlusThreeHD_TFIDF_usingDAPs1_clean.rds', 'RData', 'HTA4_29_50001,HTA4_29_50002,HTA4_29_50003,HTA4_29_50004,HTA4_29_50005,HTA4_29_50006,HTA4_29_50007,HTA4_29_50008,HTA4_29_50009,HTA4_29_50010,HTA4_29_50011,HTA4_29_50012,HTA4_29_50013,HTA4_29_50014,HTA4_29_50015,HTA4_29_50016,HTA4_29_50017,HTA4_29_50018,HTA4_28_50019,HTA4_28_50020,HTA4_28_50021,HTA4_28_50022,HTA4_28_50023,HTA4_28_50024,HTA4_28_50025,HTA4_28_50026,HTA4_28_50027,HTA4_28_50028,HTA4_28_50029,HTA4_28_50030,HTA4_28_50031,HTA4_28_50032,HTA4_28_50033,HTA4_28_50034,HTA4_28_50035,HTA4_28_50036,HTA4_30_50037,HTA4_30_50038,HTA4_30_50039,HTA4_30_50040,HTA4_30_50041,HTA4_30_50042,HTA4_30_50043,HTA4_30_50044,HTA4_30_50045,HTA4_30_50046,HTA4_30_50047,HTA4_30_50048,HTA4_30_50049,HTA4_30_50050,HTA4_30_50051,HTA4_30_50052,HTA4_30_50053,HTA4_30_50054', 'HTA4_0_80004', 'Other', 'Similar to integrating data from 18 patients, we first merged the peaks from all healthy donors using the mergePeaks module in scATAC-pro and reconstructed the peak-by-cell count matrix using combined peaks and the reConstMtx module in scATAC-pro. The 21 (7 healthy donors each with 3 sorted samples) peak-by-cell count matrices were then combined into a bigger peak-by-cell count matrix and converted to a Seurat object. We next applied the same procedure as integrating patient scATAC-Seq data, except using the first 30 PCs.', 'Other', 'https://doi.org/10.1182/blood.2021013442', 'syn51301153', 'HTAN CHOP', 'Release 3.0', 2, '8de923b2-cc77-4cff-a251-82c8671941a4', 4349730506, 'c673923b2f72e7b4027a118904fda907', NULL), ('ScATAC-seqLevel4', 'single_cell_ATACseq_level_4_clean/umap_coordinates_scATAC_healthydonor.tsv', 'tsv', 'HTA4_29_50001,HTA4_29_50002,HTA4_29_50003,HTA4_29_50004,HTA4_29_50005,HTA4_29_50006,HTA4_29_50007,HTA4_29_50008,HTA4_29_50009,HTA4_29_50010,HTA4_29_50011,HTA4_29_50012,HTA4_29_50013,HTA4_29_50014,HTA4_29_50015,HTA4_29_50016,HTA4_29_50017,HTA4_29_50018,HTA4_28_50019,HTA4_28_50020,HTA4_28_50021,HTA4_28_50022,HTA4_28_50023,HTA4_28_50024,HTA4_28_50025,HTA4_28_50026,HTA4_28_50027,HTA4_28_50028,HTA4_28_50029,HTA4_28_50030,HTA4_28_50031,HTA4_28_50032,HTA4_28_50033,HTA4_28_50034,HTA4_28_50035,HTA4_28_50036,HTA4_30_50037,HTA4_30_50038,HTA4_30_50039,HTA4_30_50040,HTA4_30_50041,HTA4_30_50042,HTA4_30_50043,HTA4_30_50044,HTA4_30_50045,HTA4_30_50046,HTA4_30_50047,HTA4_30_50048,HTA4_30_50049,HTA4_30_50050,HTA4_30_50051,HTA4_30_50052,HTA4_30_50053,HTA4_30_50054', 'HTA4_0_80007', 'Other', 'Similar to integrating data from 18 patients, we first merged the peaks from all healthy donors using the mergePeaks module in scATAC-pro and reconstructed the peak-by-cell count matrix using combined peaks and the reConstMtx module in scATAC-pro. The 21 (7 healthy donors each with 3 sorted samples) peak-by-cell count matrices were then combined into a bigger peak-by-cell count matrix and converted to a Seurat object. We next applied the same procedure as integrating patient scATAC-Seq data, except using the first 30 PCs.', 'Other', 'https://doi.org/10.1182/blood.2021013442', 'syn51301055', 'HTAN CHOP', 'Release 3.0', 2, '3678cda0-673e-4338-aad2-452d9cc4a19f', 2141213, '8f25a84d529ba6b93948f4849e47f7fc', NULL), ('ScATAC-seqLevel4', 'single_cell_ATACseq_level_4_clean/metadata_scATAC_patient.tsv', 'tsv', 'HTA4_4_50055,HTA4_4_50056,HTA4_4_50057,HTA4_4_50058,HTA4_4_50059,HTA4_4_50060,HTA4_4_50061,HTA4_4_50062,HTA4_4_50063,HTA4_4_50064,HTA4_4_50065,HTA4_4_50066,HTA4_5_50067,HTA4_5_50068,HTA4_5_50069,HTA4_5_50070,HTA4_5_50071,HTA4_5_50072,HTA4_11_50073,HTA4_11_50074,HTA4_11_50075,HTA4_11_50076,HTA4_11_50077,HTA4_11_50078,HTA4_8_50079,HTA4_8_50080,HTA4_8_50081,HTA4_8_50082,HTA4_8_50083,HTA4_8_50084,HTA4_12_50085,HTA4_12_50086,HTA4_12_50087,HTA4_12_50088,HTA4_12_50089,HTA4_12_50090,HTA4_3_50091,HTA4_3_50092,HTA4_3_50093,HTA4_3_50094,HTA4_3_50095,HTA4_3_50096,HTA4_1_50097,HTA4_1_50098,HTA4_1_50099,HTA4_1_50100,HTA4_1_50101,HTA4_1_50102,HTA4_7_50103,HTA4_7_50104,HTA4_7_50105,HTA4_7_50106,HTA4_7_50107,HTA4_7_50108,HTA4_13_50109,HTA4_13_50110,HTA4_13_50111,HTA4_13_50112,HTA4_13_50113,HTA4_13_50114,HTA4_15_50115,HTA4_15_50116,HTA4_15_50117,HTA4_15_50118,HTA4_15_50119,HTA4_15_50120,HTA4_10_50121,HTA4_10_50122,HTA4_10_50123,HTA4_10_50124,HTA4_10_50125,HTA4_10_50126,HTA4_17_50127,HTA4_17_50128,HTA4_17_50129,HTA4_17_50130,HTA4_17_50131,HTA4_17_50132,HTA4_18_50133,HTA4_18_50134,HTA4_18_50135,HTA4_18_50136,HTA4_18_50137,HTA4_18_50138,HTA4_16_50139,HTA4_16_50140,HTA4_16_50141,HTA4_16_50142,HTA4_16_50143,HTA4_16_50144,HTA4_9_50145,HTA4_9_50146,HTA4_9_50147,HTA4_9_50148,HTA4_9_50149,HTA4_9_50150,HTA4_6_50151,HTA4_6_50152,HTA4_6_50153,HTA4_6_50154,HTA4_6_50155,HTA4_6_50156,HTA4_14_50157,HTA4_14_50158,HTA4_14_50159,HTA4_14_50160,HTA4_14_50161,HTA4_14_50162,HTA4_2_50163,HTA4_2_50164,HTA4_2_50165,HTA4_2_50166,HTA4_2_50167,HTA4_2_50168', 'HTA4_0_80002', 'Other', 'To integrate data from all patients, we first merged peaks called from all patients using the mergePeaks module of scATAC-pro. With this union set of peaks, we reconstructed the peak-by-cell count matrix for each patient using the reConstMtx module in scATAC- pro. The 18 peak-by-cell matrices were combined into a bigger peak-by-cell matrix and converted to a Seurat object. The data were next normalized using the TF-IDF normalization in scATAC-pro. Top 10,000 variably accessible peaks (VAPs) were identified using the FindVariableFeatures function in Seurat. We further filtered peaks that were accessible in fewer than 1% of the cells. TF-IDF normalization was then performed on the count matrix with only the VAPs. The PCA was performed on the VAPs and the cells were clustered using the first 50 PCs and the FindClusters function in Seurat with the resolution=0.4. To reduce batch effect, we reselect the VAPs across the clusters. To do so, we summarized the count matrix into a peak-by-cluster count matrix and normalized it using the cpm function in the edgeR package with parameter prior.count = 1. The top 10,000 VAPs across the normalized peak-by-cluster count matrix were selected and scaled, followed by PCA and clustering analyses. This process of re-selecting VAPs was repeated twice and the resulting first 50 PCs were used in Uniform Manifold Approximation and Projection (UMAP) using the RunUMAP function in Seurat for visualization with the default setting.', 'Other', 'https://doi.org/10.1182/blood.2021013442', 'syn51229375', 'HTAN CHOP', 'Release 3.0', 2, '9ac8c5a8-d40a-48d0-bf95-21983a73d370', 7643086, '12890902d15a95561b16ce2234b6496c', NULL), ('ScATAC-seqLevel4', 'single_cell_ATACseq_level_4_clean/metadata_scATAC_patient_normcell.tsv', 'tsv', 'HTA4_4_50055,HTA4_4_50056,HTA4_4_50057,HTA4_4_50058,HTA4_4_50059,HTA4_4_50060,HTA4_4_50061,HTA4_4_50062,HTA4_4_50063,HTA4_4_50064,HTA4_4_50065,HTA4_4_50066,HTA4_5_50067,HTA4_5_50068,HTA4_5_50069,HTA4_5_50070,HTA4_5_50071,HTA4_5_50072,HTA4_11_50073,HTA4_11_50074,HTA4_11_50075,HTA4_11_50076,HTA4_11_50077,HTA4_11_50078,HTA4_8_50079,HTA4_8_50080,HTA4_8_50081,HTA4_8_50082,HTA4_8_50083,HTA4_8_50084,HTA4_12_50085,HTA4_12_50086,HTA4_12_50087,HTA4_12_50088,HTA4_12_50089,HTA4_12_50090,HTA4_3_50091,HTA4_3_50092,HTA4_3_50093,HTA4_3_50094,HTA4_3_50095,HTA4_3_50096,HTA4_1_50097,HTA4_1_50098,HTA4_1_50099,HTA4_1_50100,HTA4_1_50101,HTA4_1_50102,HTA4_7_50103,HTA4_7_50104,HTA4_7_50105,HTA4_7_50106,HTA4_7_50107,HTA4_7_50108,HTA4_13_50109,HTA4_13_50110,HTA4_13_50111,HTA4_13_50112,HTA4_13_50113,HTA4_13_50114,HTA4_15_50115,HTA4_15_50116,HTA4_15_50117,HTA4_15_50118,HTA4_15_50119,HTA4_15_50120,HTA4_10_50121,HTA4_10_50122,HTA4_10_50123,HTA4_10_50124,HTA4_10_50125,HTA4_10_50126,HTA4_17_50127,HTA4_17_50128,HTA4_17_50129,HTA4_17_50130,HTA4_17_50131,HTA4_17_50132,HTA4_18_50133,HTA4_18_50134,HTA4_18_50135,HTA4_18_50136,HTA4_18_50137,HTA4_18_50138,HTA4_16_50139,HTA4_16_50140,HTA4_16_50141,HTA4_16_50142,HTA4_16_50143,HTA4_16_50144,HTA4_9_50145,HTA4_9_50146,HTA4_9_50147,HTA4_9_50148,HTA4_9_50149,HTA4_9_50150,HTA4_6_50151,HTA4_6_50152,HTA4_6_50153,HTA4_6_50154,HTA4_6_50155,HTA4_6_50156,HTA4_14_50157,HTA4_14_50158,HTA4_14_50159,HTA4_14_50160,HTA4_14_50161,HTA4_14_50162,HTA4_2_50163,HTA4_2_50164,HTA4_2_50165,HTA4_2_50166,HTA4_2_50167,HTA4_2_50168', 'HTA4_0_80003', 'Other', 'To integrate data from all patients, we first merged peaks called from all patients using the mergePeaks module of scATAC-pro. With this union set of peaks, we reconstructed the peak-by-cell count matrix for each patient using the reConstMtx module in scATAC- pro. The 18 peak-by-cell matrices were combined into a bigger peak-by-cell matrix and converted to a Seurat object. The data were next normalized using the TF-IDF normalization in scATAC-pro. Top 10,000 variably accessible peaks (VAPs) were identified using the FindVariableFeatures function in Seurat. We further filtered peaks that were accessible in fewer than 1% of the cells. TF-IDF normalization was then performed on the count matrix with only the VAPs. The PCA was performed on the VAPs and the cells were clustered using the first 50 PCs and the FindClusters function in Seurat with the resolution=0.4. To reduce batch effect, we reselect the VAPs across the clusters. To do so, we summarized the count matrix into a peak-by-cluster count matrix and normalized it using the cpm function in the edgeR package with parameter prior.count = 1. The top 10,000 VAPs across the normalized peak-by-cluster count matrix were selected and scaled, followed by PCA and clustering analyses. This process of re-selecting VAPs was repeated twice and the resulting first 50 PCs were used in Uniform Manifold Approximation and Projection (UMAP) using the RunUMAP function in Seurat for visualization with the default setting.', 'Other', 'https://doi.org/10.1182/blood.2021013442', 'syn51229367', 'HTAN CHOP', 'Release 3.0', 2, '6b88a343-4570-4afa-a804-3b7c8a4506c1', 1545355, 'acd926ae918210a288612b6a5033ff1a', NULL);