coyotte508 HF staff commited on
Commit
aa125df
β€’
1 Parent(s): 74532d2

πŸ› Fix export of convos (#267)

Browse files
Files changed (1) hide show
  1. src/routes/admin/export/+server.ts +52 -1
src/routes/admin/export/+server.ts CHANGED
@@ -57,13 +57,64 @@ export async function POST({ request }) {
57
  updated_at: Date;
58
  messages: Message[];
59
  }>([
60
- { $match: { shareConversationsWithModelAuthors: true } },
 
 
 
 
 
 
61
  {
62
  $lookup: {
63
  from: "conversations",
64
  localField: "sessionId",
65
  foreignField: "sessionId",
66
  as: "conversations",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67
  pipeline: [{ $match: { model } }],
68
  },
69
  },
 
57
  updated_at: Date;
58
  messages: Message[];
59
  }>([
60
+ {
61
+ $match: {
62
+ shareConversationsWithModelAuthors: true,
63
+ sessionId: { $exists: true },
64
+ userId: { $exists: false },
65
+ },
66
+ },
67
  {
68
  $lookup: {
69
  from: "conversations",
70
  localField: "sessionId",
71
  foreignField: "sessionId",
72
  as: "conversations",
73
+ pipeline: [{ $match: { model, userId: { $exists: false } } }],
74
+ },
75
+ },
76
+ { $unwind: "$conversations" },
77
+ {
78
+ $project: {
79
+ title: "$conversations.title",
80
+ created_at: "$conversations.createdAt",
81
+ updated_at: "$conversations.updatedAt",
82
+ messages: "$conversations.messages",
83
+ },
84
+ },
85
+ ])) {
86
+ await writer.appendRow({
87
+ title: conversation.title,
88
+ created_at: conversation.created_at,
89
+ updated_at: conversation.updated_at,
90
+ messages: conversation.messages.map((message: Message) => ({
91
+ from: message.from,
92
+ content: message.content,
93
+ ...(message.score ? { score: message.score } : undefined),
94
+ })),
95
+ });
96
+ ++count;
97
+
98
+ if (count % 1_000 === 0) {
99
+ console.log("Exported", count, "conversations");
100
+ }
101
+ }
102
+
103
+ console.log("exporting convos with userId");
104
+
105
+ for await (const conversation of collections.settings.aggregate<{
106
+ title: string;
107
+ created_at: Date;
108
+ updated_at: Date;
109
+ messages: Message[];
110
+ }>([
111
+ { $match: { shareConversationsWithModelAuthors: true, userId: { $exists: true } } },
112
+ {
113
+ $lookup: {
114
+ from: "conversations",
115
+ localField: "userId",
116
+ foreignField: "userId",
117
+ as: "conversations",
118
  pipeline: [{ $match: { model } }],
119
  },
120
  },