konstantgr/themisto huggingface dataset:
import pandas as pd


user_id_col = []
session_num_col = []
action_time_col = []
action_name_col = []

for (name, actions) in parsed:
    curr_session = 0
    if name in session_num:
        curr_session = session_num[name] + 1
    session_num[name] = curr_session + 1
    
    for (action_name, time) in actions:
        user_id_col.append(name)
        session_num_col.append(cur_sesstion)
        action_time_col.append(time)
        action_name_col.append(action_name)

df = pd.DataFrame({'user_id' : user_id_col, 
                   'session_num' : session_num_col, 
                   'action_time' : action_time_col, 
                   'action_name': action_name_col})
        
df.head()
import pandas as pd


user_id_col = []
session_num_col = []
action_time_col = []
action_name_col = []

for (name, actions) in parsed:
    curr_session = 0
    if name in session_num:
        curr_session = session_num[name] + 1
    session_num[name] = curr_session + 1
    
    for (action_name, time) in actions:
        user_id_col.append(name)
        session_num_col.append(curr_session)
        action_time_col.append(time)
        action_name_col.append(action_name)

df = pd.DataFrame({'user_id' : user_id_col, 
                   'session_num' : session_num_col, 
                   'action_time' : action_time_col, 
                   'action_name': action_name_col})
        
df.head()
session_num_col[:100]
import pandas as pd

session_num = dict()

user_id_col = []
session_num_col = []
action_time_col = []
action_name_col = []

for (name, actions) in parsed:
    curr_session = 0
    if name in session_num:
        curr_session = session_num[name] + 1
    session_num[name] = curr_session + 1
    
    for (action_name, time) in actions:
        user_id_col.append(name)
        session_num_col.append(curr_session)
        action_time_col.append(time)
        action_name_col.append(action_name)

df = pd.DataFrame({'user_id' : user_id_col, 
                   'session_num' : session_num_col, 
                   'action_time' : action_time_col, 
                   'action_name': action_name_col})
        
df.head()
df.head(100)
df.head(20)
import pandas as pd

session_num = dict()

user_id_col = []
session_num_col = []
action_time_col = []
action_name_col = []

for (name, actions) in parsed:
    curr_session = 0
    if name in session_num:
        curr_session = session_num[name] + 1
    session_num[name] = curr_session + 1
    
    for (action_name, time) in actions:
        user_id_col.append(name)
        session_num_col.append(curr_session)
        action_time_col.append(time)
        action_name_col.append(action_name)

df = pd.DataFrame({'user_id' : user_id_col, 
                   'session_num' : session_num_col, 
                   'action_time' : action_time_col, 
                   'action_name': action_name_col})

df
df.session_num
cnt = 0

for line in lines:
    cnt += line.count('@User122')
cnt
cnt = 0

for line in lines:
    cnt += line.count('@User112')
cnt
cnt = 0

for (name, act) in parsed:
    if name == '@user112':
        cnt++

cnt
cnt = 0

for (name, act) in parsed:
    if name == '@user112':
        cnt += 1

cnt
cnt = 0

for (name, act) in parsed:
    if name == '@User112':
        cnt += 1

cnt
session_num["@User112"]
import pandas as pd

session_num = dict()

user_id_col = []
session_num_col = []
action_time_col = []
action_name_col = []

for (name, actions) in parsed:
    curr_session = 0
    if name in session_num:
        curr_session = session_num[name] + 1
    session_num[name] = curr_session
    
    for (action_name, time) in actions:
        user_id_col.append(name)
        session_num_col.append(curr_session)
        action_time_col.append(time)
        action_name_col.append(action_name)

df = pd.DataFrame({'user_id' : user_id_col, 
                   'session_num' : session_num_col, 
                   'action_time' : action_time_col, 
                   'action_name': action_name_col})

df
df.session_num.max
df.session_num.max()
df['user_id'].value_counts().numpy()
df['user_id'].value_counts().tonumpy()
df['user_id'].value_counts().mean()
df['user_id'].value_counts().max()
df.groupby("user_id").mean()
df.groupby("user_id").count()
df['action_time'].day_name()
df['action_time'][0]
df['action_time'][0].day_name()
df['action_time'][0].data()
df['action_time'][0].date()
df['action_time'].apply(date)
df['action_time'].date()
df['action_time'].apply(Timestamp.date)
from pandas import Timestamp

df['action_time'].apply(Timestamp.date)
df['user_id', 'action_time'].apply(lambda row: row.action_time = row.action_time.date())
df['user_id', 'action_time'].apply(lambda row: 
                                       row.action_time = row.action_time.date()
                                       row)
df['user_id', 'action_time'].dt.date()
df[action_time'].dt.date()
df['action_time'].dt.date()
df2 = df['user_id', 'action_time']
df2['action_time'] = df2['action_time'].apply(Timestamp.date)
df2
df2 = df['user_id', 'action_time']
# df2['action_time'] = df2['action_time'].apply(Timestamp.date)
# df2
df2 = df['action_time']
# df2['action_time'] = df2['action_time'].apply(Timestamp.date)
# df2
df2 = df['action_time']
# df2['action_time'] = df2['action_time'].apply(Timestamp.date)
df2
df2 = df['user_id']
# df2['action_time'] = df2['action_time'].apply(Timestamp.date)
df2
df2 = df['user_id', 'action_time']
# df2['action_time'] = df2['action_time'].apply(Timestamp.date)
df2
df2 = df[['user_id', 'action_time']
# df2['action_time'] = df2['action_time'].apply(Timestamp.date)
df2
df2 = df[['user_id', 'action_time']]
# df2['action_time'] = df2['action_time'].apply(Timestamp.date)
df2
df2 = df[['user_id', 'action_time']]
df2['action_time'] = df2['action_time'].apply(Timestamp.date)
df2
df2 = df[['user_id', 'action_time']]
df2['action_time'].apply(Timestamp.date)
df2
df2 = df[['user_id', 'action_time']]
df2.loc['action_time'] = df2['action_time'].apply(Timestamp.date)
df2
df2 = df[['user_id', 'action_time']]
df2['action_time'] = df2['action_time'].apply(Timestamp.date)
df2.groupby('action_time').count()
df2.groupby('action_time').count()
df2.groupby(['action_time', 'user_id']).count()
df2.groupby(['user_id', 'action_time']).count()
df2.groupby(['user_id', 'action_time'])
parsed = sorted(parsed, key = lambda x: x[1][0][1])
parsed[0]
parsed[0][1]
parsed[0][1][0]
parsed[0][1][0][1]
parsed = sorted(parsed, key = lambda x: print(x))
for (name, act) in parsed:
    if (len(act) == 0):
        print(name)
parsed = list(map(parse, lines))
parsed = filter(labmda x: len(x[1]) > 0, parsed)
len(parsed)
parsed = list(map(parse, lines))
parsed = filter(lambda x: len(x[1]) > 0, parsed)
len(parsed)
parsed = list(map(parse, lines))
parsed = list(filter(lambda x: len(x[1]) > 0, parsed))
len(parsed)
import re
from datetime import datetime

def parse(log):
    name = log.split()[0][:-1]
    log = log[len(name) + 2:]
    log = re.split('-+>', log)
    
    actions = []
    
    for s in log:
        s = s.strip()
        if (len(s) > 0):
            action_name = s.split()[0]
            time = s[len(action_name):].strip()
            if time[0] != '(':
                time = '(' + time
            if time == "(null)" or action_name == 'END':
                continue
            
            time = datetime.strptime(time.strip(), '(%d/%m/%y | %H:%M:%S)')
            actions.append((action_name, time))
    return name, actions
parsed[-1]
df2 = df[['user_id', 'action_time']]
df2['action_time'] = df2['action_time'].apply(Timestamp.date)
from pandas import Timestamp
df2 = df[['user_id', 'action_time']]
df2.loc['action_time'] = df2['action_time'].apply(Timestamp.date)
df2 = df[['user_id', 'action_time']].copy()
df2.loc['action_time'] = df2['action_time'].apply(Timestamp.date)
df2 = df[['user_id', 'action_time']].copy()
df2.loc['action_time'] = df2['action_time'].apply(Timestamp.date)
df2
df2.groupby(['user_id', 'action_time']).size()
df2.groupby(['user_id', 'action_time']).size().groupby('user_id').mean()
df2.groupby(['user_id', 'action_time']).size().mean()
df2 = df[['user_id', 'action_time']].copy()
df2['action_time'] = df2['action_time'].apply(Timestamp.date)
df2
df2 = df.groupby(['user_id', 'session_num']).apply(lambda x: x.sort_values('action_time'))

count = 0
for _, group in df.groupby(['user_id', 'session_num']):
    if any((group['action_name'] == val).rolling(3).sum().max() == 3 for val in group['action_name'].unique()):
        count += 1

print(count)
df2 = df.groupby(['user_id', 'session_num']).apply(lambda x: x.sort_values('action_time'))

count = 0
for _, group in df2.groupby(['user_id', 'session_num']):
    if any((group['action_name'] == val).rolling(3).sum().max() == 3 for val in group['action_name'].unique()):
        count += 1

print(count)
count = 0
for _, group in df.groupby(['user_id', 'session_num']):
    if any((group['action_name'] == val).rolling(3).sum().max() == 3 for val in group['action_name'].unique()):
        count += 1

print(count)
from tqdm.notebook.tqdm import iqdm
!pip install tqdm
from tqdm.notebook import iqdm
from tqdm.notebook import tqdm
count = 0
for _, group in tqdm(df.groupby(['user_id', 'session_num'])):
    group = group.sort_values('action_time')
    
    if any((group['action_name'] == val).rolling(3).sum().max() == 3 for val in group['action_name'].unique()):
        count += 1

print(count)
count_all = df.groupby(['user_id', 'session_num']).ngroups
count / count_all
deltas = []

for _, group in tqdm(df.groupby(['user_id', 'session_num'])):
    group = group.sort_values('action_time')
    
    for _, actions in group.groupby('action_name'):
        
        print(actions)
        1 / 0
deltas = []

for _, group in tqdm(df.groupby(['user_id', 'session_num'])):
    group = group.sort_values('action_time')
    
    for _, actions in group.groupby('action_name'):
        
        print(actions['action_time '])
        1 / 0
deltas = []

for _, group in tqdm(df.groupby(['user_id', 'session_num'])):
    group = group.sort_values('action_time')
    
    for _, actions in group.groupby('action_name'):
        
        print(actions['action_time'].to_numpy())
        1 / 0
# Example dataframe with sorted times
df = pd.DataFrame({
    'time': pd.to_datetime(['2023-05-01 10:00:00', '2023-05-01 10:05:00', '2023-05-01 10:10:00',
                            '2023-05-01 10:20:00', '2023-05-01 10:30:00'])
})

# Calculate deltas in time
deltas = df['time'].diff()

print(deltas)
# Example dataframe with sorted times
df = pd.DataFrame({
    'time': pd.to_datetime(['2023-05-01 10:00:00', '2023-05-01 10:05:00', '2023-05-01 10:10:00',
                            '2023-05-01 10:20:00', '2023-05-01 10:30:00'])
})

# Calculate deltas in time
deltas = df['time'].diff()

print(df, deltas)
# Example dataframe with sorted times
df = pd.DataFrame({
    'time': pd.to_datetime(['2023-05-01 10:00:00', '2023-05-01 10:05:00', '2023-05-01 10:10:00',
                            '2023-05-01 10:20:00', '2023-05-01 10:30:00'])
})

# Calculate deltas in time
deltas = df['time'].diff()

print(df)
print(deltas)
# Example dataframe with sorted times
df = pd.DataFrame({
    'time': pd.to_datetime(['2023-05-01 10:00:00', '2023-05-01 10:05:00', '2023-05-01 10:10:00',
                            '2023-05-01 10:20:00', '2023-05-01 10:30:00'])
})

# Calculate deltas in time
deltas = df['time'].diff().to_numpy()

print(df)
print(deltas)
# Example dataframe with sorted times
df = pd.DataFrame({
    'time': pd.to_datetime(['2023-05-01 10:00:00', '2023-05-01 10:05:00', '2023-05-01 10:10:00',
                            '2023-05-01 10:20:00', '2023-05-01 10:30:00'])
})

# Calculate deltas in time
deltas = df['time'].diff().to_numpy()[1:]

print(df)
print(deltas)
# Example dataframe with sorted times
df = pd.DataFrame({
    'time': pd.to_datetime(['2023-05-01 10:00:00', '2023-05-01 10:05:00', '2023-05-01 10:10:00',
                            '2023-05-01 10:20:00', '2023-05-02 10:30:00'])
})

# Calculate deltas in time
deltas = df['time'].diff().to_numpy()[1:]

print(df)
print(deltas)
# Example dataframe with sorted times
df = pd.DataFrame({
    'time': pd.to_datetime(['2023-05-01 10:00:00', '2023-05-01 10:00:01', '2023-05-01 10:10:00',
                            '2023-05-01 10:20:00', '2023-05-01 10:30:00'])
})

# Calculate deltas in time
deltas = df['time'].diff().to_numpy()[1:]

print(df)
print(deltas)
deltas = []

for _, group in tqdm(df.groupby(['user_id', 'session_num'])):
    group = group.sort_values('action_time')
    
    for _, actions in group.groupby('action_name'):
        
        print(actions['action_time'].diff().to_numpy()[1:] / 1e9)
        1 / 0
deltas = []

for _, group in tqdm(df.groupby(['user_id', 'session_num'])):
    group = group.sort_values('action_time')
    
    for _, actions in group.groupby('action_name'):
        
        print(actions)
        
        print(actions['action_time'].diff().to_numpy()[1:] / 1e9)
        1 / 0
deltas = []

for _, group in tqdm(df.groupby(['user_id', 'session_num'])):
    group = group.sort_values('action_time')
    
    for _, actions in group.groupby('action_name'):
        

        
        for delta in actions['action_time'].diff().to_numpy()[1:] / 1e9:
            deltas.append(delta)
len(deltas)
deltas = []

for _, group in tqdm(df.groupby(['user_id', 'session_num', 'action_name'])):
    group = group.sort_values('action_time')
    
    for delta in group['action_time'].diff().to_numpy()[1:] / 1e9:
        deltas.append(delta)
deltas = []


df2 = df.sort_values('action_time')


for _, group in tqdm(df2.groupby(['user_id', 'session_num', 'action_name'])):
    
    for delta in group['action_time'].diff().to_numpy()[1:] / 1e9:
        deltas.append(delta)
deltas = []

df2 = df.sort_values('action_time')

df2.groupby(['user_id', 'session_num', 'action_name'])['action_time']

# for _, group in tqdm(df2.groupby(['user_id', 'session_num', 'action_name'])):
    
#     for delta in group['action_time'].diff().to_numpy()[1:] / 1e9:
#         deltas.append(delta)
deltas = []

df2 = df.sort_values('action_time')

df2.groupby(['user_id', 'session_num', 'action_name'])['action_time'].diff()

# for _, group in tqdm(df2.groupby(['user_id', 'session_num', 'action_name'])):
    
#     for delta in group['action_time'].diff().to_numpy()[1:] / 1e9:
#         deltas.append(delta)
deltas = []

df2 = df.sort_values('action_time')

for _, group in tqdm(df2.groupby(['user_id', 'session_num', 'action_name'])):
    
    for delta in group['action_time'].diff().to_numpy()[1:] / 1e9:
        deltas.append(delta)
import matplotlib.pyplot as plt

plt.hist(deltas, bins=10)
plt.show()
import numpy as np

np.quantile(deltas, 0.95)
!pip install matplotlib
deltas.int()
deltas.to(int)
deltas.type(int)
deltas.stype(int)
deltas=deltas.astype(int)
"avav"
df.groupby(['action_time', 'action_name']).size()
deltas= np.array(deltas).astype(int)
'asdfadsfs'
deltas = []

df2 = df.sort_values('action_time')

for _, group in tqdm(df2.groupby(['user_id', 'session_num', 'action_name'])):
    
    for delta in group['action_time'].diff().to_numpy()[1:] / 1e9:
        deltas.append(delta)
        
deltas= np.array(deltas).astype(int)
deltas.shape