#!/usr/bin/env python3
import re
import pandas as pd

# Define a function to extract time and message number from subscriber log content
def extract_subscriber_data(log_file):
    time_and_message_numbers = []
    message_number = 0
    message_size = 0
    with open(log_file, 'r') as f:
        for line in f:
            match = re.search(r"\[ROS-SUBSCRIBE\(us\)\](\d+)", line)
            if match:
                next_line = next(f)
                match2 = re.search(r"(\d+):Received message on topic", next_line)
                time_to_subscribe = int(match.group(1))
                if match2:
                    message_number = int(match2.group(1))
                    next_line1 = next(f)
                    match3 = re.search(r"\(Size: (\d+)\)", next_line1)
                    if match3:
                        message_size = int(match3.group(1))
                    else:
                        print("no match found")
                else:
                    print("no match found")
                time_and_message_numbers.append((message_number, time_to_subscribe, message_size))
    return time_and_message_numbers

# Define a function to extract message number and time from publisher log content
def extract_publisher_data(log_file):
    data = []
    with open(log_file, 'r') as f:
        for line in f:
            match = re.search(r"(\d+):Publishing data from the source path", line)
            if match:
                next_line = next(f)
                match3 = re.search(r"\[ROS-PUBLISH\(us\)\](\d+)", next_line)
                message_number = int(match.group(1))
                if match3:
                    time = int(match3.group(1))
                else:
                    print("no match found")
                data.append((message_number, time))
    return data

# Define a function to save data to an Excel file
def save_to_excel(subscriber_data, publisher_data, output_file):
    # Create DataFrames
    df_subscriber = pd.DataFrame(subscriber_data, columns=['Subscriber Message Number', 'Subscriber Time', 'Message Size'])
    df_publisher = pd.DataFrame(publisher_data, columns=['Publisher Message Number', 'Publisher Time'])

    # Combine DataFrames
    df_combined = pd.concat([df_subscriber, df_publisher], axis=1)

    # Save the DataFrame to an Excel file
    df_combined.to_excel(output_file, index=False)

# Main execution
if __name__ == "__main__":
    # Get the input file names from the user
    subscriber_log_file = input("Enter the subscriber log file name: ")
    publisher_log_file = input("Enter the publisher log file name: ")

    # Output file name
    output_file = "extracted_time_data.xlsx"

    # Extract data
    subscriber_data = extract_subscriber_data(subscriber_log_file)
    publisher_data = extract_publisher_data(publisher_log_file)

    # Save data to Excel
    save_to_excel(subscriber_data, publisher_data, output_file)

    print(f"Data extracted from {subscriber_log_file} and {publisher_log_file} and stored in {output_file}")




With reference script above I need a script to extract the Subscriber Message Number', 'Subscriber Time' and Publisher Message Number', 'Publisher Time' from the below type of log files and then  save it in a similar fashion as above script.

The example log is given below.

Publisher log:


[2024-09-19 10:53:32] root@v4x:~/IT_MultiTest/image_transfer# 
[2024-09-19 10:53:33] root@v4x:~/IT_MultiTest/image_transfer# ./ros2_image_publisher_v4h2 30 topic1
[2024-09-19 10:54:56] Running with 30 FPS, Publishing to topic: topic1
[2024-09-19 10:54:56] 2024-09-19 05:25:24.616 [XMLPARSER ] Given max_message_size not in between 0 - 67108860 bytes, So the value 67108860 chosen as default -> Function check_parameter_boundary
[2024-09-19 10:54:56] 2024-09-19 05:25:24.617 [PCIE_PORT_MANAGER ] Port ID generated 7413 -> Function get_unique_port
[2024-09-19 10:54:56] 2024-09-19 05:25:24.617 [RTPS_PARTICIPANT_ATTRIBUTES ] Builtin transport LARGE_DATA_PCIE configuration enabled -> Function setup_transports_large_data_pcie
[2024-09-19 10:54:56] 2024-09-19 05:25:24.618 [SHM_TRANSPORT ] Initialization -> Function init
[2024-09-19 10:54:56] 2024-09-19 05:25:24.619 [PCIE_TRANSPORT ] Initialization -> Function init
[2024-09-19 10:54:56] 2024-09-19 05:25:24.619 [PCIE_INTERFACE_MMAP ] Sendbuff mapping success -> Function init_pcie
[2024-09-19 10:54:56] 2024-09-19 05:25:24.619 [PCIE_INTERFACE_MMAP ] recvbuff mapping success -> Function init_pcie
[2024-09-19 10:54:56] 2024-09-19 05:25:24.620 [UDP_TRANSPORT ] Initialization -> Function init
[2024-09-19 10:54:57] Publishing Frame Id: 1
[2024-09-19 10:54:57] [ROS-PUBLISH(us)]1726723525624697
[2024-09-19 10:54:57] Publishing Frame Id: 2
[2024-09-19 10:54:57] [ROS-PUBLISH(us)]1726723525785044
[2024-09-19 10:54:57] Publishing Frame Id: 3
[2024-09-19 10:54:57] [ROS-PUBLISH(us)]1726723525890694
[2024-09-19 10:54:57] Publishing Frame Id: 4
[2024-09-19 10:54:57] [ROS-PUBLISH(us)]1726723526012588
[2024-09-19 10:54:57] Publishing Frame Id: 5
[2024-09-19 10:54:57] [ROS-PUBLISH(us)]1726723526121595
[2024-09-19 10:54:57] Publishing Frame Id: 6
[2024-09-19 10:54:57] [ROS-PUBLISH(us)]1726723526231662
[2024-09-19 10:54:57] Publishing Frame Id: 7
[2024-09-19 10:54:57] [ROS-PUBLISH(us)]1726723526353745
[2024-09-19 10:54:58] Publishing Frame Id: 8
[2024-09-19 10:54:58] [INFO] [1726723526.458721373] [image_publisher]: FPS: 6.788169
[2024-09-19 10:54:58] [ROS-PUBLISH(us)]1726723526458962
[2024-09-19 10:54:58] Publishing Frame Id: 9
[2024-09-19 10:54:58] [ROS-PUBLISH(us)]1726723526565301
[2024-09-19 10:54:58] Publishing Frame Id: 10
[2024-09-19 10:54:58] [ROS-PUBLISH(us)]1726723526739712
[2024-09-19 10:54:58] Publishing Frame Id: 11
[2024-09-19 10:54:58] [ROS-PUBLISH(us)]1726723526933838
[2024-09-19 10:54:58] Publishing Frame Id: 12
[2024-09-19 10:54:58] [ROS-PUBLISH(us)]1726723527120336
[2024-09-19 10:54:58] Publishing Frame Id: 13
[2024-09-19 10:54:58] [ROS-PUBLISH(us)]1726723527315331
[2024-09-19 10:54:59] Publishing Frame Id: 14
[2024-09-19 10:54:59] [INFO] [1726723527.500686116] [image_publisher]: FPS: 5.758178
[2024-09-19 10:54:59] [ROS-PUBLISH(us)]1726723527500862
[2024-09-19 10:54:59] Publishing Frame Id: 15
[2024-09-19 10:54:59] [ROS-PUBLISH(us)]1726723527686525
[2024-09-19 10:54:59] Publishing Frame Id: 16
[2024-09-19 10:54:59] [ROS-PUBLISH(us)]1726723527881831
[2024-09-19 10:54:59] Publishing Frame Id: 17
[2024-09-19 10:54:59] [ROS-PUBLISH(us)]1726723528111252
[2024-09-19 10:54:59] Publishing Frame Id: 18
[2024-09-19 10:54:59] [ROS-PUBLISH(us)]1726723528235989
[2024-09-19 10:54:59] Publishing Frame Id: 19
[2024-09-19 10:54:59] [ROS-PUBLISH(us)]1726723528349318
[2024-09-19 10:55:00] Publishing Frame Id: 20
[2024-09-19 10:55:00] [ROS-PUBLISH(us)]1726723528461303
[2024-09-19 10:55:00] Publishing Frame Id: 21
[2024-09-19 10:55:00] [INFO] [1726723528.571387294] [image_publisher]: FPS: 6.537773
[2024-09-19 10:55:00] [ROS-PUBLISH(us)]1726723528571560
[2024-09-19 10:55:00] Publishing Frame Id: 22
[2024-09-19 10:55:00] [ROS-PUBLISH(us)]1726723528679479
[2024-09-19 10:55:00] Publishing Frame Id: 23
[2024-09-19 10:55:00] [ROS-PUBLISH(us)]1726723528787739
[2024-09-19 10:55:00] Publishing Frame Id: 24
[2024-09-19 10:55:00] [ROS-PUBLISH(us)]1726723528899038
[2024-09-19 10:55:00] Publishing Frame Id: 25
[2024-09-19 10:55:00] [ROS-PUBLISH(us)]1726723529004400
[2024-09-19 10:55:00] Publishing Frame Id: 26
[2024-09-19 10:55:00] [ROS-PUBLISH(us)]1726723529113083
[2024-09-19 10:55:00] Publishing Frame Id: 27
[2024-09-19 10:55:00] [ROS-PUBLISH(us)]1726723529288057
[2024-09-19 10:55:01] Publishing Frame Id: 28
[2024-09-19 10:55:01] [ROS-PUBLISH(us)]1726723529476661
[2024-09-19 10:55:01] Publishing Frame Id: 29
[2024-09-19 10:55:01] [INFO] [1726723529.673494383] [image_publisher]: FPS: 7.258817
[2024-09-19 10:55:01] [ROS-PUBLISH(us)]1726723529673662
[2024-09-19 10:55:01] Publishing Frame Id: 30
[2024-09-19 10:55:01] [ROS-PUBLISH(us)]1726723529858882
[2024-09-19 10:55:01] Publishing Frame Id: 31
[2024-09-19 10:55:01] [ROS-PUBLISH(us)]1726723530045488
[2024-09-19 10:55:01] Publishing Frame Id: 32
[2024-09-19 10:55:01] [ROS-PUBLISH(us)]1726723530234067
[2024-09-19 10:55:02] Publishing Frame Id: 33
[2024-09-19 10:55:02] [ROS-PUBLISH(us)]1726723530426346
[2024-09-19 10:55:02] Publishing Frame Id: 34
[2024-09-19 10:55:02] [ROS-PUBLISH(us)]1726723530620641
[2024-09-19 10:55:02] Publishing Frame Id: 35
[2024-09-19 10:55:02] [INFO] [1726723530.745657604] [image_publisher]: FPS: 5.596164
[2024-09-19 10:55:02] [ROS-PUBLISH(us)]1726723530745866
[2024-09-19 10:55:02] Publishing Frame Id: 36
[2024-09-19 10:55:02] [ROS-PUBLISH(us)]1726723530857686
[2024-09-19 10:55:02] Publishing Frame Id: 37
[2024-09-19 10:55:02] [ROS-PUBLISH(us)]1726723530967899
[2024-09-19 10:55:02] Publishing Frame Id: 38
[2024-09-19 10:55:02] [ROS-PUBLISH(us)]1726723531075614
[2024-09-19 10:55:02] Publishing Frame Id: 39
[2024-09-19 10:55:02] [ROS-PUBLISH(us)]1726723531187520
[2024-09-19 10:55:02] Publishing Frame Id: 40
[2024-09-19 10:55:02] [ROS-PUBLISH(us)]1726723531299870
[2024-09-19 10:55:02] Publishing Frame Id: 41
[2024-09-19 10:55:03] [ROS-PUBLISH(us)]1726723531405860
[2024-09-19 10:55:03] Publishing Frame Id: 42
[2024-09-19 10:55:03] [ROS-PUBLISH(us)]1726723531520796
[2024-09-19 10:55:03] Publishing Frame Id: 43
[2024-09-19 10:55:03] [ROS-PUBLISH(us)]1726723531642684
[2024-09-19 10:55:03] Publishing Frame Id: 44
[2024-09-19 10:55:03] [INFO] [1726723531.817621218] [image_publisher]: FPS: 8.395795
[2024-09-19 10:55:03] [ROS-PUBLISH(us)]1726723531817838
[2024-09-19 10:55:03] Publishing Frame Id: 45
[2024-09-19 10:55:03] [ROS-PUBLISH(us)]1726723532010406
[2024-09-19 10:55:03] Publishing Frame Id: 46
[2024-09-19 10:55:03] [ROS-PUBLISH(us)]1726723532197890
[2024-09-19 10:55:03] Publishing Frame Id: 47
[2024-09-19 10:55:04] [ROS-PUBLISH(us)]1726723532389807
[2024-09-19 10:55:04] Publishing Frame Id: 48
[2024-09-19 10:55:04] [ROS-PUBLISH(us)]1726723532579813



Subscriber log:

nit Completed

[37;1m2024-09-19 05:25:20.853 [[37;1mXMLPARSER ] [37mGiven max_message_size not in between 0 - 67108860 bytes, So the value 67108860 chosen as default[34;1m -> Function [36mcheck_parameter_boundary[m

[37;1m2024-09-19 05:25:20.855 [[37;1mPCIE_PORT_MANAGER ] [37mPort ID generated 7412[34;1m -> Function [36mget_unique_port[m

[37;1m2024-09-19 05:25:20.855 [[37;1mRTPS_PARTICIPANT_ATTRIBUTES ] [37mBuiltin transport LARGE_DATA_PCIE configuration enabled[34;1m -> Function [36msetup_transports_large_data_pcie[m

[37;1m2024-09-19 05:25:20.856 [[37;1mSHM_TRANSPORT ] [37mInitialization[34;1m -> Function [36minit[m

[37;1m2024-09-19 05:25:20.859 [[37;1mPCIE_TRANSPORT ] [37mInitialization[34;1m -> Function [36minit[m

[37;1m2024-09-19 05:25:20.876 [[37;1mPCIE_INTERFACE_MMAP ] [37mSendbuff mapping success[34;1m -> Function [36minit_pcie[m

[37;1m2024-09-19 05:25:20.894 [[37;1mPCIE_INTERFACE_MMAP ] [37mrecvbuff mapping success[34;1m -> Function [36minit_pcie[m

[37;1m2024-09-19 05:25:20.894 [[37;1mUDP_TRANSPORT ] [37mInitialization[34;1m -> Function [36minit[m

Subscribing to topic: topic1

Subscribed Image Frame Id :1





[ROS-SUBSCRIBE(us)]1726723525657344

FPS: 0.20079

Subscribed Image Frame Id :2





[ROS-SUBSCRIBE(us)]1726723525808079

Subscribed Image Frame Id :3





[ROS-SUBSCRIBE(us)]1726723525917535

Subscribed Image Frame Id :4





[ROS-SUBSCRIBE(us)]1726723526032479

Subscribed Image Frame Id :5





[ROS-SUBSCRIBE(us)]1726723526145056

Subscribed Image Frame Id :6





[ROS-SUBSCRIBE(us)]1726723526259187

Subscribed Image Frame Id :7





[ROS-SUBSCRIBE(us)]1726723526373499

Subscribed Image Frame Id :8





[ROS-SUBSCRIBE(us)]1726723526478892

Subscribed Image Frame Id :9





[ROS-SUBSCRIBE(us)]1726723526589475

Subscribed Image Frame Id :10





[ROS-SUBSCRIBE(us)]1726723526784271

FPS: 7.98611

Subscribed Image Frame Id :11





[ROS-SUBSCRIBE(us)]1726723526969437

Subscribed Image Frame Id :12





[ROS-SUBSCRIBE(us)]1726723527157957

Subscribed Image Frame Id :13





[ROS-SUBSCRIBE(us)]1726723527348180

Subscribed Image Frame Id :14





[ROS-SUBSCRIBE(us)]1726723527534783

Subscribed Image Frame Id :15





[ROS-SUBSCRIBE(us)]1726723527720290

Subscribed Image Frame Id :16





[ROS-SUBSCRIBE(us)]1726723527918500

FPS: 5.29007

Subscribed Image Frame Id :17





[ROS-SUBSCRIBE(us)]1726723528145656

Subscribed Image Frame Id :18





[ROS-SUBSCRIBE(us)]1726723528257615

Subscribed Image Frame Id :19





[ROS-SUBSCRIBE(us)]1726723528368901

Subscribed Image Frame Id :20





[ROS-SUBSCRIBE(us)]1726723528482517

Subscribed Image Frame Id :21





[ROS-SUBSCRIBE(us)]1726723528593476

Subscribed Image Frame Id :22





[ROS-SUBSCRIBE(us)]1726723528701174

Subscribed Image Frame Id :23





[ROS-SUBSCRIBE(us)]1726723528812653

Subscribed Image Frame Id :24





[ROS-SUBSCRIBE(us)]1726723528918321

Subscribed Image Frame Id :25





[ROS-SUBSCRIBE(us)]1726723529024210

FPS: 8.13943

Subscribed Image Frame Id :26





[ROS-SUBSCRIBE(us)]1726723529135520

Subscribed Image Frame Id :27





[ROS-SUBSCRIBE(us)]1726723529322435

Subscribed Image Frame Id :28





[ROS-SUBSCRIBE(us)]1726723529517100

Subscribed Image Frame Id :29





[ROS-SUBSCRIBE(us)]1726723529707366

Subscribed Image Frame Id :30





[ROS-SUBSCRIBE(us)]1726723529892547

Subscribed Image Frame Id :31