Skip to content

Commit b851c35

Browse files
arvinxxarvinxx
authoredNov 6, 2024··
🗃️ refactor: update database fields (#4626)
* 🗃️ refactor: update database fields * ✅ test: fix tests
·
v1.74.1v1.28.1
1 parent 4f50dfa commit b851c35

18 files changed

+3264
-169
lines changed
 

‎src/database/server/migrations/0010_add_accessed_at_and_clean_tables.sql

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
DROP TABLE "agents_tags" CASCADE;--> statement-breakpoint
2+
DROP TABLE "market" CASCADE;--> statement-breakpoint
3+
DROP TABLE "plugins" CASCADE;--> statement-breakpoint
4+
DROP TABLE "plugins_tags" CASCADE;--> statement-breakpoint
5+
DROP TABLE "tags" CASCADE;--> statement-breakpoint
6+
ALTER TABLE "agents" ADD COLUMN "accessed_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
7+
ALTER TABLE "agents_files" ADD COLUMN "accessed_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
8+
ALTER TABLE "agents_knowledge_bases" ADD COLUMN "accessed_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
9+
ALTER TABLE "async_tasks" ADD COLUMN "accessed_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
10+
ALTER TABLE "files" ADD COLUMN "accessed_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
11+
ALTER TABLE "global_files" ADD COLUMN "accessed_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
12+
ALTER TABLE "knowledge_bases" ADD COLUMN "accessed_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
13+
ALTER TABLE "messages" ADD COLUMN "accessed_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
14+
ALTER TABLE "chunks" ADD COLUMN "accessed_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
15+
ALTER TABLE "unstructured_chunks" ADD COLUMN "accessed_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
16+
ALTER TABLE "rag_eval_dataset_records" ADD COLUMN "accessed_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
17+
ALTER TABLE "rag_eval_dataset_records" ADD COLUMN "updated_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
18+
ALTER TABLE "rag_eval_datasets" ADD COLUMN "accessed_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
19+
ALTER TABLE "rag_eval_evaluations" ADD COLUMN "accessed_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
20+
ALTER TABLE "rag_eval_evaluation_records" ADD COLUMN "accessed_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
21+
ALTER TABLE "rag_eval_evaluation_records" ADD COLUMN "updated_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
22+
ALTER TABLE "session_groups" ADD COLUMN "accessed_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
23+
ALTER TABLE "sessions" ADD COLUMN "accessed_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
24+
ALTER TABLE "topics" ADD COLUMN "accessed_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
25+
ALTER TABLE "user_installed_plugins" ADD COLUMN "accessed_at" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint
26+
ALTER TABLE "users" ADD COLUMN "accessed_at" timestamp with time zone DEFAULT now() NOT NULL;

‎src/database/server/migrations/meta/0010_snapshot.json

+3,184
Large diffs are not rendered by default.

‎src/database/server/migrations/meta/_journal.json

+7
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,13 @@
7070
"when": 1729699958471,
7171
"tag": "0009_remove_unused_user_tables",
7272
"breakpoints": true
73+
},
74+
{
75+
"idx": 10,
76+
"version": "7",
77+
"when": 1730900133049,
78+
"tag": "0010_add_accessed_at_and_clean_tables",
79+
"breakpoints": true
7380
}
7481
],
7582
"version": "6"

‎src/database/server/models/__tests__/session.test.ts

-2
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import {
99
agents,
1010
agentsToSessions,
1111
messages,
12-
plugins,
1312
sessionGroups,
1413
sessions,
1514
topics,
@@ -30,7 +29,6 @@ const userId = 'session-user';
3029
const sessionModel = new SessionModel(userId);
3130

3231
beforeEach(async () => {
33-
await serverDB.delete(plugins);
3432
await serverDB.delete(users);
3533
// 并创建初始用户
3634
await serverDB.insert(users).values({ id: userId });

‎src/database/server/models/__tests__/topic.test.ts

+2
Original file line numberDiff line numberDiff line change
@@ -430,6 +430,7 @@ describe('TopicModel', () => {
430430
clientId: null,
431431
createdAt: expect.any(Date),
432432
updatedAt: expect.any(Date),
433+
accessedAt: expect.any(Date),
433434
});
434435

435436
// 断言 topic 已在数据库中创建
@@ -476,6 +477,7 @@ describe('TopicModel', () => {
476477
userId,
477478
createdAt: expect.any(Date),
478479
updatedAt: expect.any(Date),
480+
accessedAt: expect.any(Date),
479481
});
480482

481483
// 断言 topic 已在数据库中创建

‎src/database/server/schemas/lobechat/_helpers.ts

+8
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,11 @@ export const timestamptz = (name: string) => timestamp(name, { withTimezone: tru
44

55
export const createdAt = () => timestamptz('created_at').notNull().defaultNow();
66
export const updatedAt = () => timestamptz('updated_at').notNull().defaultNow();
7+
export const accessedAt = () => timestamptz('accessed_at').notNull().defaultNow();
8+
9+
// columns.helpers.ts
10+
export const timestamps = {
11+
accessedAt: accessedAt(),
12+
createdAt: createdAt(),
13+
updatedAt: updatedAt(),
14+
};

‎src/database/server/schemas/lobechat/agent.ts

+6-7
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { createInsertSchema } from 'drizzle-zod';
55
import { LobeAgentChatConfig, LobeAgentTTSConfig } from '@/types/agent';
66

77
import { idGenerator, randomSlug } from '../../utils/idGenerator';
8-
import { createdAt, updatedAt } from './_helpers';
8+
import { timestamps } from './_helpers';
99
import { files, knowledgeBases } from './file';
1010
import { users } from './user';
1111

@@ -41,8 +41,7 @@ export const agents = pgTable('agents', {
4141
systemRole: text('system_role'),
4242
tts: jsonb('tts').$type<LobeAgentTTSConfig>(),
4343

44-
createdAt: createdAt(),
45-
updatedAt: updatedAt(),
44+
...timestamps,
4645
});
4746

4847
export const insertAgentSchema = createInsertSchema(agents);
@@ -63,8 +62,8 @@ export const agentsKnowledgeBases = pgTable(
6362
.references(() => users.id, { onDelete: 'cascade' })
6463
.notNull(),
6564
enabled: boolean('enabled').default(true),
66-
createdAt: createdAt(),
67-
updatedAt: updatedAt(),
65+
66+
...timestamps,
6867
},
6968
(t) => ({
7069
pk: primaryKey({ columns: [t.agentId, t.knowledgeBaseId] }),
@@ -84,8 +83,8 @@ export const agentsFiles = pgTable(
8483
userId: text('user_id')
8584
.references(() => users.id, { onDelete: 'cascade' })
8685
.notNull(),
87-
createdAt: createdAt(),
88-
updatedAt: updatedAt(),
86+
87+
...timestamps,
8988
},
9089
(t) => ({
9190
pk: primaryKey({ columns: [t.fileId, t.agentId, t.userId] }),

‎src/database/server/schemas/lobechat/asyncTask.ts

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/* eslint-disable sort-keys-fix/sort-keys-fix */
22
import { integer, jsonb, pgTable, text, uuid } from 'drizzle-orm/pg-core';
33

4-
import { createdAt, updatedAt } from './_helpers';
4+
import { timestamps } from './_helpers';
55
import { users } from './user';
66

77
export const asyncTasks = pgTable('async_tasks', {
@@ -16,8 +16,7 @@ export const asyncTasks = pgTable('async_tasks', {
1616
.notNull(),
1717
duration: integer('duration'),
1818

19-
createdAt: createdAt(),
20-
updatedAt: updatedAt(),
19+
...timestamps,
2120
});
2221

2322
export type NewAsyncTaskItem = typeof asyncTasks.$inferInsert;

‎src/database/server/schemas/lobechat/discover.ts

-84
This file was deleted.

‎src/database/server/schemas/lobechat/file.ts

+5-5
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import {
1212
import { createInsertSchema } from 'drizzle-zod';
1313

1414
import { idGenerator } from '../../utils/idGenerator';
15-
import { createdAt, updatedAt } from './_helpers';
15+
import { accessedAt, createdAt, timestamps } from './_helpers';
1616
import { asyncTasks } from './asyncTask';
1717
import { chunks } from './rag';
1818
import { users } from './user';
@@ -23,7 +23,9 @@ export const globalFiles = pgTable('global_files', {
2323
size: integer('size').notNull(),
2424
url: text('url').notNull(),
2525
metadata: jsonb('metadata'),
26+
2627
createdAt: createdAt(),
28+
accessedAt: accessedAt(),
2729
});
2830

2931
export type NewGlobalFile = typeof globalFiles.$inferInsert;
@@ -51,8 +53,7 @@ export const files = pgTable('files', {
5153
onDelete: 'set null',
5254
}),
5355

54-
createdAt: createdAt(),
55-
updatedAt: updatedAt(),
56+
...timestamps,
5657
});
5758

5859
export type NewFile = typeof files.$inferInsert;
@@ -91,8 +92,7 @@ export const knowledgeBases = pgTable('knowledge_bases', {
9192

9293
settings: jsonb('settings'),
9394

94-
createdAt: createdAt(),
95-
updatedAt: updatedAt(),
95+
...timestamps,
9696
});
9797

9898
export const insertKnowledgeBasesSchema = createInsertSchema(knowledgeBases);

‎src/database/server/schemas/lobechat/index.ts

-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
export * from './agent';
22
export * from './asyncTask';
3-
export * from './discover';
43
export * from './file';
54
export * from './message';
65
export * from './nextauth';

‎src/database/server/schemas/lobechat/message.ts

+2-3
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import {
1313
import { createSelectSchema } from 'drizzle-zod';
1414

1515
import { idGenerator } from '../../utils/idGenerator';
16-
import { createdAt, updatedAt } from './_helpers';
16+
import { timestamps } from './_helpers';
1717
import { agents } from './agent';
1818
import { files } from './file';
1919
import { chunks, embeddings } from './rag';
@@ -58,8 +58,7 @@ export const messages = pgTable(
5858
// used for group chat
5959
agentId: text('agent_id').references(() => agents.id, { onDelete: 'set null' }),
6060

61-
createdAt: createdAt(),
62-
updatedAt: updatedAt(),
61+
...timestamps,
6362
},
6463
(table) => ({
6564
createdAtIdx: index('messages_created_at_idx').on(table.createdAt),

‎src/database/server/schemas/lobechat/rag.ts

+4-6
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/* eslint-disable sort-keys-fix/sort-keys-fix */
22
import { integer, jsonb, pgTable, text, uuid, varchar, vector } from 'drizzle-orm/pg-core';
33

4-
import { createdAt, updatedAt } from './_helpers';
4+
import { timestamps } from './_helpers';
55
import { files } from './file';
66
import { users } from './user';
77

@@ -13,10 +13,9 @@ export const chunks = pgTable('chunks', {
1313
index: integer('index'),
1414
type: varchar('type'),
1515

16-
createdAt: createdAt(),
17-
updatedAt: updatedAt(),
18-
1916
userId: text('user_id').references(() => users.id, { onDelete: 'cascade' }),
17+
18+
...timestamps,
2019
});
2120

2221
export type NewChunkItem = typeof chunks.$inferInsert & { fileId?: string };
@@ -28,8 +27,7 @@ export const unstructuredChunks = pgTable('unstructured_chunks', {
2827
index: integer('index'),
2928
type: varchar('type'),
3029

31-
createdAt: createdAt(),
32-
updatedAt: updatedAt(),
30+
...timestamps,
3331

3432
parentId: varchar('parent_id'),
3533
compositeId: uuid('composite_id').references(() => chunks.id, { onDelete: 'cascade' }),

‎src/database/server/schemas/lobechat/ragEvals.ts

+5-7
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { integer, jsonb, pgTable, text, uuid } from 'drizzle-orm/pg-core';
44
import { DEFAULT_EMBEDDING_MODEL, DEFAULT_MODEL } from '@/const/settings';
55
import { EvalEvaluationStatus } from '@/types/eval';
66

7-
import { createdAt, updatedAt } from './_helpers';
7+
import { timestamps } from './_helpers';
88
import { knowledgeBases } from './file';
99
import { embeddings } from './rag';
1010
import { users } from './user';
@@ -20,8 +20,7 @@ export const evalDatasets = pgTable('rag_eval_datasets', {
2020
}),
2121
userId: text('user_id').references(() => users.id, { onDelete: 'cascade' }),
2222

23-
updatedAt: updatedAt(),
24-
createdAt: createdAt(),
23+
...timestamps,
2524
});
2625

2726
export type NewEvalDatasetsItem = typeof evalDatasets.$inferInsert;
@@ -39,7 +38,7 @@ export const evalDatasetRecords = pgTable('rag_eval_dataset_records', {
3938
metadata: jsonb('metadata'),
4039

4140
userId: text('user_id').references(() => users.id, { onDelete: 'cascade' }),
42-
createdAt: createdAt(),
41+
...timestamps,
4342
});
4443

4544
export type NewEvalDatasetRecordsItem = typeof evalDatasetRecords.$inferInsert;
@@ -64,8 +63,7 @@ export const evalEvaluation = pgTable('rag_eval_evaluations', {
6463
embeddingModel: text('embedding_model').$defaultFn(() => DEFAULT_EMBEDDING_MODEL),
6564

6665
userId: text('user_id').references(() => users.id, { onDelete: 'cascade' }),
67-
createdAt: createdAt(),
68-
updatedAt: updatedAt(),
66+
...timestamps,
6967
});
7068

7169
export type NewEvalEvaluationItem = typeof evalEvaluation.$inferInsert;
@@ -98,7 +96,7 @@ export const evaluationRecords = pgTable('rag_eval_evaluation_records', {
9896
.notNull(),
9997

10098
userId: text('user_id').references(() => users.id, { onDelete: 'cascade' }),
101-
createdAt: createdAt(),
99+
...timestamps,
102100
});
103101

104102
export type NewEvaluationRecordsItem = typeof evaluationRecords.$inferInsert;

‎src/database/server/schemas/lobechat/relations.ts

-33
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import { pgTable, primaryKey, text } from 'drizzle-orm/pg-core';
44

55
import { agents, agentsFiles, agentsKnowledgeBases } from './agent';
66
import { asyncTasks } from './asyncTask';
7-
import { agentsTags, plugins, pluginsTags, tags } from './discover';
87
import { files, knowledgeBases } from './file';
98
import { messages, messagesFiles } from './message';
109
import { unstructuredChunks } from './rag';
@@ -48,26 +47,6 @@ export const topicRelations = relations(topics, ({ one }) => ({
4847
}),
4948
}));
5049

51-
export const pluginsRelations = relations(plugins, ({ many }) => ({
52-
pluginsTags: many(pluginsTags),
53-
}));
54-
55-
export const pluginsTagsRelations = relations(pluginsTags, ({ one }) => ({
56-
plugin: one(plugins, {
57-
fields: [pluginsTags.pluginId],
58-
references: [plugins.id],
59-
}),
60-
tag: one(tags, {
61-
fields: [pluginsTags.tagId],
62-
references: [tags.id],
63-
}),
64-
}));
65-
66-
export const tagsRelations = relations(tags, ({ many }) => ({
67-
agentsTags: many(agentsTags),
68-
pluginsTags: many(pluginsTags),
69-
}));
70-
7150
export const messagesRelations = relations(messages, ({ many, one }) => ({
7251
filesToMessages: many(messagesFiles),
7352

@@ -91,7 +70,6 @@ export const agentsRelations = relations(agents, ({ many }) => ({
9170
agentsToSessions: many(agentsToSessions),
9271
knowledgeBases: many(agentsKnowledgeBases),
9372
files: many(agentsFiles),
94-
agentsTags: many(agentsTags),
9573
}));
9674

9775
export const agentsToSessionsRelations = relations(agentsToSessions, ({ one }) => ({
@@ -116,17 +94,6 @@ export const agentsKnowledgeBasesRelations = relations(agentsKnowledgeBases, ({
11694
}),
11795
}));
11896

119-
export const agentsTagsRelations = relations(agentsTags, ({ one }) => ({
120-
agent: one(agents, {
121-
fields: [agentsTags.agentId],
122-
references: [agents.id],
123-
}),
124-
tag: one(tags, {
125-
fields: [agentsTags.tagId],
126-
references: [tags.id],
127-
}),
128-
}));
129-
13097
export const sessionsRelations = relations(sessions, ({ many, one }) => ({
13198
filesToSessions: many(filesToSessions),
13299
agentsToSessions: many(agentsToSessions),

‎src/database/server/schemas/lobechat/session.ts

+3-5
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { boolean, integer, pgTable, text, unique, uniqueIndex, varchar } from 'd
33
import { createInsertSchema } from 'drizzle-zod';
44

55
import { idGenerator, randomSlug } from '../../utils/idGenerator';
6-
import { createdAt, updatedAt } from './_helpers';
6+
import { timestamps } from './_helpers';
77
import { users } from './user';
88

99
// ======= sessionGroups ======= //
@@ -22,8 +22,7 @@ export const sessionGroups = pgTable(
2222
.notNull(),
2323

2424
clientId: text('client_id'),
25-
createdAt: createdAt(),
26-
updatedAt: updatedAt(),
25+
...timestamps,
2726
},
2827
(table) => ({
2928
clientIdUnique: unique('session_group_client_id_user_unique').on(table.clientId, table.userId),
@@ -60,8 +59,7 @@ export const sessions = pgTable(
6059
clientId: text('client_id'),
6160
pinned: boolean('pinned').default(false),
6261

63-
createdAt: createdAt(),
64-
updatedAt: updatedAt(),
62+
...timestamps,
6563
},
6664
(t) => ({
6765
slugUserIdUnique: uniqueIndex('slug_user_id_unique').on(t.slug, t.userId),

‎src/database/server/schemas/lobechat/topic.ts

+7-8
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,26 @@
1-
// ======== topics ======= //
1+
/* eslint-disable sort-keys-fix/sort-keys-fix */
22
import { boolean, pgTable, text, unique } from 'drizzle-orm/pg-core';
33

44
import { idGenerator } from '../../utils/idGenerator';
5-
import { createdAt, updatedAt } from './_helpers';
5+
import { timestamps } from './_helpers';
66
import { sessions } from './session';
77
import { users } from './user';
88

99
export const topics = pgTable(
1010
'topics',
1111
{
12-
clientId: text('client_id'),
13-
createdAt: createdAt(),
14-
favorite: boolean('favorite').default(false),
1512
id: text('id')
1613
.$defaultFn(() => idGenerator('topics'))
1714
.primaryKey(),
18-
sessionId: text('session_id').references(() => sessions.id, { onDelete: 'cascade' }),
1915
title: text('title'),
20-
21-
updatedAt: updatedAt(),
16+
favorite: boolean('favorite').default(false),
17+
sessionId: text('session_id').references(() => sessions.id, { onDelete: 'cascade' }),
2218
userId: text('user_id')
2319
.references(() => users.id, { onDelete: 'cascade' })
2420
.notNull(),
21+
clientId: text('client_id'),
22+
23+
...timestamps,
2524
},
2625
(t) => ({
2726
clientIdUnique: unique('topic_client_id_user_id_unique').on(t.clientId, t.userId),

‎src/database/server/schemas/lobechat/user.ts

+3-5
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { boolean, jsonb, pgTable, primaryKey, text } from 'drizzle-orm/pg-core';
55
import { DEFAULT_PREFERENCE } from '@/const/user';
66
import { CustomPluginParams } from '@/types/tool/plugin';
77

8-
import { createdAt, timestamptz, updatedAt } from './_helpers';
8+
import { timestamps, timestamptz } from './_helpers';
99

1010
export const users = pgTable('users', {
1111
id: text('id').primaryKey().notNull(),
@@ -27,8 +27,7 @@ export const users = pgTable('users', {
2727

2828
preference: jsonb('preference').$defaultFn(() => DEFAULT_PREFERENCE),
2929

30-
createdAt: createdAt(),
31-
updatedAt: updatedAt(),
30+
...timestamps,
3231
});
3332

3433
export type NewUser = typeof users.$inferInsert;
@@ -61,8 +60,7 @@ export const installedPlugins = pgTable(
6160
settings: jsonb('settings'),
6261
customParams: jsonb('custom_params').$type<CustomPluginParams>(),
6362

64-
createdAt: createdAt(),
65-
updatedAt: updatedAt(),
63+
...timestamps,
6664
},
6765
(self) => ({
6866
id: primaryKey({ columns: [self.userId, self.identifier] }),

0 commit comments

Comments
 (0)
Please sign in to comment.