Package: pydoris-client
Apache Doris Python ClientA Apache Doris client for the Python programming language.Apache Doris is a high-performance, real-time analytical database based on MPP architecture, known for its extreme speed and ease of use. It only requires a sub-second response time to return query results under massive data and can support not only high-concurrent point query scenarios but also high-throughput complex analysis scenarios. All this makes Apache Doris an ideal tool for scenarios including report analysis, ad-hoc query, unified data warehouse, and data lake query acceleration. On Apache Doris, users can build various applications, such as user behavior analysis, AB test platform, log retrieval analysis, user portrait analysis, and order analysis.Installationpip install pydoris-clientDorisClient Usagefrompydoris.doris_clientimport*frompydoris.util.generate_test_dataimport*fe_host="127.0.0.1"fe_http_port="8040"fe_query_port="9030"username='root'passwd=""db="test"doris_client=DorisClient(fe_host=fe_host,fe_query_port=fe_query_port,fe_http_port=fe_http_port,username=username,password=passwd,db=db)deftest_create_database():returndoris_client.create_database('pydoris_client_test')deftest_create_table():doris_client.execute("""create table if not exists pydoris_client_test.write_test(f_id int,f_decimal decimal(18,6),f_timestamp bigint,f_datetime datetime(6),f_str string,f_float float,f_boolean boolean)duplicate key(`f_id`)distributed by hash(`f_id`) buckets 1properties("replication_allocation" = "tag.location.default: 1");""")deftest_get_table_columns():print(doris_client.get_table_columns('pydoris_client_test','write_test'))defgen_test_data(num):list=[]start_date=datetime(2023,1,1)end_date=datetime(2023,12,31)foriinrange(num):line=(i,generate_decimal(),generate_timestamp(),generate_random_datetime(start_date,end_date),generate_random_string(20),generate_float(),generate_boolean())list.append(line)returnlist# If your data line delimiter need to be specified，use options.set_line_delimiter(delimiter)deftest_write_csv():# print(list)df=pd.DataFrame(gen_test_data(100000))df.columns=['f_id','f_decimal','f_timestamp','f_datetime','f_str','f_float','f_boolean']# doris_client.options.set_csv_format(",").set_auto_uuid_label().set_line_delimiter("\\n")csv=df.to_csv(header=False,index=False)doris_client.write("pydoris_client_test.write_test",csv)# 1. If you need use json format to insert data to Doris, you need set json format ,#    because the default format is csv format# 2. When you json data is [{},{}] please set strip_outer_array=true# 3. You can customize data import labels , use options.set_label(your_label)deftest_write_json():df=pd.DataFrame(gen_test_data(100000),columns=['f_id','f_decimal','f_timestamp','f_datetime','f_str','f_float','f_boolean'])json_data=df.to_json(orient='records')options=WriteOptions()options.set_json_format()options.set_option("strip_outer_array","true")doris_client.write("pydoris_client_test.write_test",json_data,options=options)# data_df: pd.DataFrame, table_name: str, table_model: str is must# When repeat_replacement = True, tables with duplicate names will be deleted，be carefuldeftest_write_from_df():df=pd.DataFrame(gen_test_data(100000),columns=['f_id','f_decimal','f_timestamp','f_datetime','f_str','f_float','f_boolean'])doris_client.write_from_df(df,"pydoris_client_test.df_write_test","UNIQUE",['f_id'],distributed_hash_key=["f_id"],buckets=1,field_mapping=[("f_decimal","Decimal(18,6)")],table_properties={"replication_allocation":"tag.location.default: 1"},repeat_replacement=False)deftest_read_to_df():dataframe=doris_client.query_to_dataframe("select * from pydoris_client_test.write_test limit 1000",['f_id','f_decimal','f_timestamp','f_datetime','f_str','f_float','f_boolean'])withpd.option_context('expand_frame_repr',False,'display.max_rows',None):print(dataframe)deftest_query():importdatetimestart=datetime.datetime.now()result=doris_client.query("select * from pydoris_client_test.write_test")end=datetime.datetime.now()print((end-start).seconds)print(len(result))deftest_list_tables():tables=doris_client.list_tables("pydoris_client_test")print(tables)deftest_drop_table():db='pydoris_client_test'table_name1='write_test'table_name2='df_write_test'tables=doris_client.list_tables(db)print(tables)doris_client.drop_table(db,table_name1)doris_client.drop_table(db,table_name2)tables=doris_client.list_tables(db)print(tables)if__name__=='__main__':test_create_database()test_create_table()test_get_table_columns()test_write_csv()test_write_json()test_write_from_df()test_read_to_df()test_query()test_list_tables()test_drop_table()