Package: dsmlibrary
DSM LibraryDataNodeinit DataNodefromdsmlibrary.datanodeimportDataNodedata=DataNode(token="<token>",apikey="<apikey>",dataplatform_api_uri="<dataplatform_api_uri>",object_storage_uri="<object_storage_uri>",use_env=<True/False(default(True))>)upload filedata.upload_file(directory_id=<directory_id>,file_path='<file_path>',description="<description(optional)>")download filedata.download_file(file_id=<file_id>,download_path="<place download file save> (default ./dsm.tmp)")get filemeta,file=data.get_file(file_id="<file_id>")# meta -> dict# file -> io bytes# example read csv pandasmeta,file=data.get_file(file_id="<file_id>")df=pd.read_csv(file)...read dfdf=data.read_df(file_id="<file_id>")# df return as pandas dataframeread ddf.parquet must use this functionddf=data.read_ddf(file_id="<file_id>")# ddf return as dask dataframewrite parquet filedf=...# pandas dataframe or dask dataframedata.write(df=df,directory=<directory_id>,name="<save_file_name>",description="<description>",replace=<replaceiffileexists.defaultFalse>,datadict=<TrueorFalsedefaultFalse>,profiling=<TrueorFalsedefaultFalse>,lineage=<listoffileid.eg[1,2,3]>)writeListDataNodedf=...# pandas dataframe or dask dataframedata.writeListDataNode(df=df,directory_id=<directory_id>,name="<save_file_name>",description="<description>",replace=<replaceiffileexists.defaultFalse>,datadict=<TrueorFalsedefaultFalse>,profiling=<TrueorFalsedefaultFalse>,lineage=<listoffileid.eg[1,2,3]>)get file idfile_id=data.get_file_id(name=<filename>,directory_id=<directoryid>)# file_id return int fileIDget directory iddirectory_id = data.get_directory_id(parent_dir_id=<directory id>, name=<file name>)
# directory_id return int directoryIDget get_file_versionuse for listDataNodefileVersion=data.get_file_version(file_id=<fileid>)# return dict `file_id` and `timestamp`Clickhouseimoprt data to clickhousefromdsmlibrary.clickhouseimportClickHouseddf=...# pandas dataframe or dask dataframe## to warehousetable_name=<your_table_name>partition_by=<your_partition_by>connection={'host':'','port':,'database':'','user':'','password':'','settings':{'use_numpy':True},'secure':False}warehouse=ClickHouse(connection=connection)tableName=warehouse.get_or_createTable(ddf=ddf,tableName=table_name,partition_by=partition_by)warehouse.write(ddf=ddf,tableName=tableName)query data from clickhousequery=f"""SELECT * FROM{tableName}LIMIT 10"""warehouse.read(sqlQuery=query)drop tablewarehouse.dropTable(tableName=table_name)optionaluse for custom config insert data to clickhouseconfig={'n_partition_per_block':10,'n_row_per_loop':1000}warehouse=ClickHouse(connection=connection,config=config)truncate tablewarehouse.truncateTable(tableName=table_name)APIdsmlibrarydsmlibrary.datanode.DataNodeupload_filedownload_fileread_dfread_ddfwriteget_file_iddsmlibrary.clickhouse.ClickHouseget_or_createTablewritereaddropTableUse for pipelinedata = DataNode(apikey="<APIKEY>")use api key for authenticateMDMsemantic similaritypip install "dsmlibrary[mdm]"see examplehereGendatadict PDFfromdsmlibrary.datadictimportGenerateDatadictgd=GenerateDatadict(token="<token>",apikey="<apikey>",dataplatform_api_uri="<dataplatform_api_uri>",object_storage_uri="<object_storage_uri>")gd.generate_datadict(name="<NAME>",directory_id=<DIR_IDfordatadictfile>,file_ids=[<FILE_ID>,<FILE_ID>,...])use token or apikey