diff --git a/Dockerfile b/Dockerfile index 5e879ba..af544a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,4 +29,4 @@ RUN chmod +x ./history-api ./email-worker ./storage-worker EXPOSE 3344 -CMD ["./history-api"] \ No newline at end of file +CMD ["./history-api"] diff --git a/cmd/api/main.go b/cmd/api/main.go index eb9fa35..744cf28 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -4,6 +4,7 @@ import ( "context" "fmt" _ "history-api/docs" + "history-api/pkg/ai" "history-api/pkg/cache" "history-api/pkg/config" "history-api/pkg/database" @@ -67,7 +68,6 @@ func StartServer() { } defer sqlTile.Close() - sqlRasterTile, err := mbtiles.NewMBTilesDB("data/raster.mbtiles") if err != nil { log.Error().Msg(err.Error()) @@ -93,6 +93,12 @@ func StartServer() { panic(err) } + raguUtils, err := ai.NewRagUtils() + if err != nil { + log.Error().Msg(err.Error()) + panic(err) + } + serverIp, _ := config.GetConfig("SERVER_IP") if serverIp == "" { serverIp = "127.0.0.1" @@ -104,7 +110,7 @@ func StartServer() { } serverHttp := NewHttpServer() - serverHttp.SetupServer(poolPg, sqlTile, sqlRasterTile, redisClient, storageClient, googleOAuthConfig) + serverHttp.SetupServer(poolPg, sqlTile, sqlRasterTile, redisClient, storageClient, googleOAuthConfig, raguUtils) Singleton = serverHttp done := make(chan bool, 1) diff --git a/cmd/api/server.go b/cmd/api/server.go index faf3646..0588524 100644 --- a/cmd/api/server.go +++ b/cmd/api/server.go @@ -8,6 +8,7 @@ import ( "history-api/internal/repositories" "history-api/internal/routes" "history-api/internal/services" + "history-api/pkg/ai" "history-api/pkg/cache" "history-api/pkg/storage" "os" @@ -63,6 +64,7 @@ func (s *FiberServer) SetupServer( redis cache.Cache, sclient storage.Storage, oauth *oauth2.Config, + raguUtils *ai.RagUtils, ) { // Apply CORS middleware s.App.Use(cors.New(cors.Config{ @@ -92,6 +94,8 @@ func (s *FiberServer) SetupServer( commitRepo := repositories.NewCommitRepository(poolPg, redis) submissionRepo := repositories.NewSubmissionRepository(poolPg, redis) + raguRepo := repositories.NewRagRepository(poolPg, redis) + // service setup authService := services.NewAuthService(userRepo, roleRepo, tokenRepo, redis, poolPg) userService := services.NewUserService(userRepo, roleRepo, redis, poolPg) @@ -107,8 +111,10 @@ func (s *FiberServer) SetupServer( commitService := services.NewCommitService(poolPg, commitRepo, projectRepo) submissionService := services.NewSubmissionService( submissionRepo, projectRepo, commitRepo, - userRepo, wikiRepo, geometryRepo, entityRepo, poolPg, redis, + userRepo, wikiRepo, geometryRepo, entityRepo, + raguRepo, raguUtils, poolPg, redis, ) + chatbotService := services.NewChatbotService(raguRepo, raguUtils) // controller setup authController := controllers.NewAuthController(authService, oauth) @@ -124,6 +130,7 @@ func (s *FiberServer) SetupServer( projectController := controllers.NewProjectController(projectService) commitController := controllers.NewCommitController(commitService) submissionController := controllers.NewSubmissionController(submissionService) + chatbotController := controllers.NewChatbotController(chatbotService) // route setup routes.AuthRoutes(s.App, authController, userRepo) @@ -138,5 +145,6 @@ func (s *FiberServer) SetupServer( routes.WikiRoutes(s.App, wikiController) routes.ProjectRoutes(s.App, projectController, commitController, userRepo) routes.SubmissionRoutes(s.App, submissionController, userRepo) + routes.ChatbotRoutes(s.App, chatbotController, userRepo) routes.NotFoundRoute(s.App) } diff --git a/db/migrations/0000013_rag_documents.down.sql b/db/migrations/0000013_rag_documents.down.sql new file mode 100644 index 0000000..ed58d41 --- /dev/null +++ b/db/migrations/0000013_rag_documents.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS rag_chunks; +DROP EXTENSION IF EXISTS vector; diff --git a/db/migrations/0000013_rag_documents.up.sql b/db/migrations/0000013_rag_documents.up.sql new file mode 100644 index 0000000..e2f4ea1 --- /dev/null +++ b/db/migrations/0000013_rag_documents.up.sql @@ -0,0 +1,21 @@ +CREATE EXTENSION IF NOT EXISTS vector; + +CREATE TABLE IF NOT EXISTS rag_chunks ( + id UUID PRIMARY KEY DEFAULT uuidv7(), + source_type VARCHAR(50) NOT NULL, + source_id UUID NOT NULL, + project_id UUID REFERENCES projects(id) ON DELETE CASCADE, + chunk_index INT NOT NULL, + content TEXT NOT NULL, + embedding vector(3072), + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +CREATE INDEX idx_rag_chunks_source ON rag_chunks(source_type, source_id); +CREATE INDEX idx_rag_chunks_project ON rag_chunks(project_id); + +CREATE TRIGGER trigger_rag_chunks_updated_at +BEFORE UPDATE ON rag_chunks +FOR EACH ROW +EXECUTE FUNCTION update_updated_at(); diff --git a/db/migrations/000001_users.up.sql b/db/migrations/000001_users.up.sql index 2f74ed5..dc4bebb 100644 --- a/db/migrations/000001_users.up.sql +++ b/db/migrations/000001_users.up.sql @@ -1,6 +1,7 @@ CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS pg_trgm; CREATE EXTENSION IF NOT EXISTS btree_gist; +CREATE EXTENSION IF NOT EXISTS vector; CREATE TABLE IF NOT EXISTS users ( id UUID PRIMARY KEY DEFAULT uuidv7(), diff --git a/db/migrations/000008_wiki.up.sql b/db/migrations/000008_wiki.up.sql index 8ddab87..db1e7b6 100644 --- a/db/migrations/000008_wiki.up.sql +++ b/db/migrations/000008_wiki.up.sql @@ -4,7 +4,7 @@ CREATE TABLE IF NOT EXISTS wikis ( id UUID PRIMARY KEY DEFAULT uuidv7(), project_id UUID NOT NULL REFERENCES projects(id) ON DELETE CASCADE, title TEXT, - content JSONB, + content TEXT, is_deleted BOOLEAN NOT NULL DEFAULT false, created_at TIMESTAMPTZ DEFAULT now(), updated_at TIMESTAMPTZ DEFAULT now() diff --git a/db/query/rag.sql b/db/query/rag.sql new file mode 100644 index 0000000..311e0bf --- /dev/null +++ b/db/query/rag.sql @@ -0,0 +1,24 @@ +-- name: CreateRagChunk :one +INSERT INTO rag_chunks ( + id, source_type, source_id, project_id, chunk_index, content, embedding +) VALUES ( + COALESCE(sqlc.narg('id')::uuid, uuidv7()), + $1, $2, $3, $4, $5, $6 +) +RETURNING *; + +-- name: SearchRagChunks :many +SELECT + id, source_type, source_id, project_id, chunk_index, content, + (1 - (embedding <=> sqlc.arg('embedding')))::float8 AS similarity +FROM rag_chunks +WHERE 1=1 + AND (sqlc.narg('project_id')::uuid IS NULL OR project_id = sqlc.narg('project_id')::uuid) + AND (sqlc.narg('source_type')::varchar IS NULL OR source_type = sqlc.narg('source_type')::varchar) + AND (1 - (embedding <=> sqlc.arg('embedding')))::float8 >= sqlc.arg('match_threshold')::float8 +ORDER BY embedding <=> sqlc.arg('embedding') +LIMIT sqlc.arg('match_count'); + +-- name: DeleteRagChunksBySourceIDs :exec +DELETE FROM rag_chunks +WHERE source_type = $1 AND source_id = ANY($2::uuid[]); diff --git a/db/schema.sql b/db/schema.sql index 46b951c..122fea0 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -101,7 +101,7 @@ CREATE TABLE IF NOT EXISTS wikis ( id UUID PRIMARY KEY DEFAULT uuidv7(), project_id UUID NOT NULL REFERENCES projects(id) ON DELETE CASCADE, title TEXT, - content JSONB, + content TEXT, is_deleted BOOLEAN NOT NULL DEFAULT false, created_at TIMESTAMPTZ DEFAULT now(), updated_at TIMESTAMPTZ DEFAULT now() @@ -172,3 +172,16 @@ CREATE TABLE IF NOT EXISTS project_members ( created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), PRIMARY KEY (project_id, user_id) ); + +CREATE TABLE IF NOT EXISTS rag_chunks ( + id UUID PRIMARY KEY DEFAULT uuidv7(), + source_type VARCHAR(50) NOT NULL, + source_id UUID NOT NULL, + project_id UUID REFERENCES projects(id) ON DELETE CASCADE, + chunk_index INT NOT NULL, + content TEXT NOT NULL, + embedding vector(3072), + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + diff --git a/deployment/app/Dockerfile b/deployment/app/Dockerfile new file mode 100644 index 0000000..af544a8 --- /dev/null +++ b/deployment/app/Dockerfile @@ -0,0 +1,32 @@ +FROM golang:1.26.1-alpine AS builder + +RUN apk add --no-cache git + +WORKDIR /app + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o history-api ./cmd/api +RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o email-worker ./cmd/worker/email +RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o storage-worker ./cmd/worker/storage + +FROM alpine:latest + +RUN apk --no-cache add ca-certificates tzdata +ENV TZ=Asia/Ho_Chi_Minh + +WORKDIR /app + +COPY --from=builder /app/history-api . +COPY --from=builder /app/email-worker . +COPY --from=builder /app/storage-worker . +COPY data ./data + +RUN chmod +x ./history-api ./email-worker ./storage-worker + +EXPOSE 3344 + +CMD ["./history-api"] diff --git a/deployment/postgres/Dockerfile b/deployment/postgres/Dockerfile new file mode 100644 index 0000000..f8f73a0 --- /dev/null +++ b/deployment/postgres/Dockerfile @@ -0,0 +1,21 @@ +FROM postgis/postgis:18-3.6 + +# Install build dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + postgresql-server-dev-18 \ + git \ + ca-certificates && \ + rm -rf /var/lib/apt/lists/* + +# Build and install pgvector from source +RUN cd /tmp && \ + git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git && \ + cd pgvector && \ + make && \ + make install && \ + rm -rf /tmp/pgvector + +# Cleanup +RUN apt-get purge -y --auto-remove build-essential postgresql-server-dev-18 git diff --git a/docker-compose.yml b/docker-compose.yml index d5bdf31..1a3a3f6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: history_db: - image: postgis/postgis:18-3.6 + image: azenkain/postgres-postgis-pgvector:18 container_name: history_db restart: unless-stopped env_file: diff --git a/docs/docs.go b/docs/docs.go index d8820d9..5cef8d7 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -399,6 +399,69 @@ const docTemplate = `{ } } }, + "/chatbot/chat": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Ask a history question based on project context or global knowledge using RAG", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Chatbot" + ], + "summary": "Ask the AI chatbot", + "parameters": [ + { + "description": "Chatbot query", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/history-api_internal_dtos_request.ChatbotDto" + } + } + ], + "responses": { + "200": { + "description": "Successful response with AI answer", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/history-api_internal_dtos_response.CommonResponse" + }, + { + "type": "object", + "properties": { + "data": { + "type": "string" + } + } + } + ] + } + }, + "400": { + "description": "Invalid request", + "schema": { + "$ref": "#/definitions/history-api_internal_dtos_response.CommonResponse" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/history-api_internal_dtos_response.CommonResponse" + } + } + } + } + }, "/entities": { "get": { "description": "Search entities with cursor pagination", @@ -3582,6 +3645,20 @@ const docTemplate = `{ } } }, + "history-api_internal_dtos_request.ChatbotDto": { + "type": "object", + "required": [ + "question" + ], + "properties": { + "project_id": { + "type": "string" + }, + "question": { + "type": "string" + } + } + }, "history-api_internal_dtos_request.CommitSnapshot": { "type": "object", "properties": { @@ -3814,8 +3891,11 @@ const docTemplate = `{ 1 ] }, - "type_id": { - "type": "string" + "time_end": { + "type": "number" + }, + "time_start": { + "type": "number" } } }, @@ -3830,7 +3910,6 @@ const docTemplate = `{ "type": "string" }, "is_deleted": { - "description": "Legacy / Compatibility", "type": "integer", "enum": [ 0, @@ -4267,10 +4346,7 @@ const docTemplate = `{ ], "properties": { "doc": { - "type": "array", - "items": { - "type": "integer" - } + "type": "string" }, "id": { "type": "string" diff --git a/docs/swagger.json b/docs/swagger.json index a9759ee..b02b389 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -392,6 +392,69 @@ } } }, + "/chatbot/chat": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Ask a history question based on project context or global knowledge using RAG", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Chatbot" + ], + "summary": "Ask the AI chatbot", + "parameters": [ + { + "description": "Chatbot query", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/history-api_internal_dtos_request.ChatbotDto" + } + } + ], + "responses": { + "200": { + "description": "Successful response with AI answer", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/history-api_internal_dtos_response.CommonResponse" + }, + { + "type": "object", + "properties": { + "data": { + "type": "string" + } + } + } + ] + } + }, + "400": { + "description": "Invalid request", + "schema": { + "$ref": "#/definitions/history-api_internal_dtos_response.CommonResponse" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/history-api_internal_dtos_response.CommonResponse" + } + } + } + } + }, "/entities": { "get": { "description": "Search entities with cursor pagination", @@ -3575,6 +3638,20 @@ } } }, + "history-api_internal_dtos_request.ChatbotDto": { + "type": "object", + "required": [ + "question" + ], + "properties": { + "project_id": { + "type": "string" + }, + "question": { + "type": "string" + } + } + }, "history-api_internal_dtos_request.CommitSnapshot": { "type": "object", "properties": { @@ -3807,8 +3884,11 @@ 1 ] }, - "type_id": { - "type": "string" + "time_end": { + "type": "number" + }, + "time_start": { + "type": "number" } } }, @@ -3823,7 +3903,6 @@ "type": "string" }, "is_deleted": { - "description": "Legacy / Compatibility", "type": "integer", "enum": [ 0, @@ -4260,10 +4339,7 @@ ], "properties": { "doc": { - "type": "array", - "items": { - "type": "integer" - } + "type": "string" }, "id": { "type": "string" diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 576fa4e..f31dfa8 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -59,6 +59,15 @@ definitions: required: - role_ids type: object + history-api_internal_dtos_request.ChatbotDto: + properties: + project_id: + type: string + question: + type: string + required: + - question + type: object history-api_internal_dtos_request.CommitSnapshot: properties: editor_feature_collection: @@ -217,8 +226,10 @@ definitions: - 0 - 1 type: integer - type_id: - type: string + time_end: + type: number + time_start: + type: number required: - id type: object @@ -227,7 +238,6 @@ definitions: entity_id: type: string is_deleted: - description: Legacy / Compatibility enum: - 0 - 1 @@ -530,9 +540,7 @@ definitions: history-api_internal_dtos_request.WikiSnapshot: properties: doc: - items: - type: integer - type: array + type: string id: type: string operation: @@ -857,6 +865,44 @@ paths: summary: Verify a security token tags: - Auth + /chatbot/chat: + post: + consumes: + - application/json + description: Ask a history question based on project context or global knowledge + using RAG + parameters: + - description: Chatbot query + in: body + name: request + required: true + schema: + $ref: '#/definitions/history-api_internal_dtos_request.ChatbotDto' + produces: + - application/json + responses: + "200": + description: Successful response with AI answer + schema: + allOf: + - $ref: '#/definitions/history-api_internal_dtos_response.CommonResponse' + - properties: + data: + type: string + type: object + "400": + description: Invalid request + schema: + $ref: '#/definitions/history-api_internal_dtos_response.CommonResponse' + "500": + description: Internal server error + schema: + $ref: '#/definitions/history-api_internal_dtos_response.CommonResponse' + security: + - ApiKeyAuth: [] + summary: Ask the AI chatbot + tags: + - Chatbot /entities: get: consumes: diff --git a/go.mod b/go.mod index 0f9d922..50f79df 100644 --- a/go.mod +++ b/go.mod @@ -17,9 +17,11 @@ require ( github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.8.0 github.com/joho/godotenv v1.5.1 + github.com/pgvector/pgvector-go v0.3.0 github.com/redis/go-redis/v9 v9.18.0 github.com/rs/zerolog v1.34.0 github.com/swaggo/swag v1.16.6 + github.com/tmc/langchaingo v0.1.14 github.com/wneessen/go-mail v0.7.2 golang.org/x/crypto v0.49.0 golang.org/x/oauth2 v0.36.0 @@ -28,9 +30,15 @@ require ( ) require ( + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/ai v0.7.0 // indirect + cloud.google.com/go/aiplatform v1.120.0 // indirect cloud.google.com/go/auth v0.18.2 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/longrunning v0.8.0 // indirect + cloud.google.com/go/vertexai v0.12.0 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/MicahParks/keyfunc/v2 v2.1.0 // indirect github.com/andybalholm/brotli v1.2.0 // indirect @@ -51,6 +59,7 @@ require ( github.com/aws/smithy-go v1.24.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dlclark/regexp2 v1.10.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.12 // indirect @@ -79,6 +88,7 @@ require ( github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/gofiber/schema v1.7.0 // indirect github.com/gofiber/utils/v2 v2.0.2 // indirect + github.com/google/generative-ai-go v0.15.1 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect github.com/googleapis/gax-go/v2 v2.19.0 // indirect @@ -91,12 +101,19 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/philhofer/fwd v1.2.0 // indirect + github.com/pkoukk/tiktoken-go v0.1.6 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/tinylib/msgp v1.6.3 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.69.0 // indirect + gitlab.com/golang-commonmark/html v0.0.0-20191124015941-a22733972181 // indirect + gitlab.com/golang-commonmark/linkify v0.0.0-20191026162114-a0c2df6c8f82 // indirect + gitlab.com/golang-commonmark/markdown v0.0.0-20211110145824-bf3e522c626a // indirect + gitlab.com/golang-commonmark/mdurl v0.0.0-20191124015652-932350d1cb84 // indirect + gitlab.com/golang-commonmark/puny v0.0.0-20191124015043-9f83538fa04f // indirect go.mongodb.org/mongo-driver v1.17.9 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect go.opentelemetry.io/otel v1.42.0 // indirect go.opentelemetry.io/otel/metric v1.42.0 // indirect @@ -107,7 +124,10 @@ require ( golang.org/x/net v0.52.0 // indirect golang.org/x/sys v0.42.0 // indirect golang.org/x/text v0.35.0 // indirect + golang.org/x/time v0.15.0 // indirect golang.org/x/tools v0.42.0 // indirect + google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect google.golang.org/grpc v1.79.3 // indirect google.golang.org/protobuf v1.36.11 // indirect diff --git a/go.sum b/go.sum index 4deae4e..e399099 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,23 @@ +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/ai v0.7.0 h1:P6+b5p4gXlza5E+u7uvcgYlzZ7103ACg70YdZeC6oGE= +cloud.google.com/go/ai v0.7.0/go.mod h1:7ozuEcraovh4ABsPbrec3o4LmFl9HigNI3D5haxYeQo= +cloud.google.com/go/aiplatform v1.120.0 h1:jKWTpEs+xoUhDa1FMdSuhMcEQYyUiMdufGyX3zvtLVQ= +cloud.google.com/go/aiplatform v1.120.0/go.mod h1:6mDthfmy0oS1EQhVFdijoxkVdI2+HIZkpuGTBpedeCg= cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM= cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= +cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= +cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk= +cloud.google.com/go/vertexai v0.12.0 h1:zTadEo/CtsoyRXNx3uGCncoWAP1H2HakGqwznt+iMo8= +cloud.google.com/go/vertexai v0.12.0/go.mod h1:8u+d0TsvBfAAd2x5R6GMgbYhsLgo3J7lmP4bR8g2ig8= +entgo.io/ent v0.14.3 h1:wokAV/kIlH9TeklJWGGS7AYJdVckr0DloWjIcO9iIIQ= +entgo.io/ent v0.14.3/go.mod h1:aDPE/OziPEu8+OWbzy4UlvWmD2/kbRuWfK2A40hcxJM= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/MicahParks/keyfunc/v2 v2.1.0 h1:6ZXKb9Rp6qp1bDbJefnG7cTH8yMN1IC/4nf+GVjO99k= @@ -54,14 +68,23 @@ github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/Tv1FZd4SCg8axKApyNyRsAt/w= +github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= +github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g= +github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98= +github.com/envoyproxy/protoc-gen-validate v1.3.0 h1:TvGH1wof4H33rezVKWSpqKz5NXWg5VPuZ0uONDT6eb4= +github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= @@ -91,7 +114,7 @@ github.com/go-openapi/spec v0.22.3 h1:qRSmj6Smz2rEBxMnLRBMeBWxbbOvuOoElvSvObIgwQ github.com/go-openapi/spec v0.22.3/go.mod h1:iIImLODL2loCh3Vnox8TY2YWYJZjMAKYyLH2Mu8lOZs= github.com/go-openapi/strfmt v0.25.0 h1:7R0RX7mbKLa9EYCTHRcCuIPcaqlyQiWNPTXwClK0saQ= github.com/go-openapi/strfmt v0.25.0/go.mod h1:nNXct7OzbwrMY9+5tLX4I21pzcmE6ccMGXl3jFdPfn8= -github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4= github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU= github.com/go-openapi/swag/fileutils v0.25.4 h1:2oI0XNW5y6UWZTC7vAxC8hmsK/tOkWXHJQH4lKjqw+Y= @@ -118,6 +141,10 @@ github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6 github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= github.com/go-openapi/validate v0.25.1 h1:sSACUI6Jcnbo5IWqbYHgjibrhhmt3vR6lCzKZnmAgBw= github.com/go-openapi/validate v0.25.1/go.mod h1:RMVyVFYte0gbSTaZ0N4KmTn6u/kClvAFp+mAVfS/DQc= +github.com/go-pg/pg/v10 v10.11.0 h1:CMKJqLgTrfpE/aOVeLdybezR2om071Vh38OLZjsyMI0= +github.com/go-pg/pg/v10 v10.11.0/go.mod h1:4BpHRoxE61y4Onpof3x1a2SQvi9c+q1dJnrNdMjsroA= +github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU= +github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -145,6 +172,8 @@ github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63Y github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/generative-ai-go v0.15.1 h1:n8aQUpvhPOlGVuM2DRkJ2jvx04zpp42B778AROJa+pQ= +github.com/google/generative-ai-go v0.15.1/go.mod h1:AAucpWZjXsDKhQYWvCYuP6d0yB1kX998pJlOW1rAesw= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= @@ -165,18 +194,26 @@ github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo= github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= +github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= -github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= @@ -186,9 +223,15 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/pgvector/pgvector-go v0.3.0 h1:Ij+Yt78R//uYqs3Zk35evZFvr+G0blW0OUN+Q2D1RWc= +github.com/pgvector/pgvector-go v0.3.0/go.mod h1:duFy+PXWfW7QQd5ibqutBO4GxLsUZ9RVXhFZGIBsWSA= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw= +github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -201,9 +244,13 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shamaton/msgpack/v3 v3.1.0 h1:jsk0vEAqVvvS9+fTZ5/EcQ9tz860c9pWxJ4Iwecz8gU= github.com/shamaton/msgpack/v3 v3.1.0/go.mod h1:DcQG8jrdrQCIxr3HlMYkiXdMhK+KfN2CitkyzsQV4uc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -212,10 +259,28 @@ github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI= github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg= github.com/tinylib/msgp v1.6.3 h1:bCSxiTz386UTgyT1i0MSCvdbWjVW+8sG3PjkGsZQt4s= github.com/tinylib/msgp v1.6.3/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/tmc/langchaingo v0.1.14 h1:o1qWBPigAIuFvrG6cjTFo0cZPFEZ47ZqpOYMjM15yZc= +github.com/tmc/langchaingo v0.1.14/go.mod h1:aKKYXYoqhIDEv7WKdpnnCLRaqXic69cX9MnDUk72378= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= +github.com/uptrace/bun v1.1.12 h1:sOjDVHxNTuM6dNGaba0wUuz7KvDE1BmNu9Gqs2gJSXQ= +github.com/uptrace/bun v1.1.12/go.mod h1:NPG6JGULBeQ9IU6yHp7YGELRa5Agmd7ATZdz4tGZ6z0= +github.com/uptrace/bun/dialect/pgdialect v1.1.12 h1:m/CM1UfOkoBTglGO5CUTKnIKKOApOYxkcP2qn0F9tJk= +github.com/uptrace/bun/dialect/pgdialect v1.1.12/go.mod h1:Ij6WIxQILxLlL2frUBxUBOZJtLElD2QQNDcu/PWDHTc= +github.com/uptrace/bun/driver/pgdriver v1.1.12 h1:3rRWB1GK0psTJrHwxzNfEij2MLibggiLdTqjTtfHc1w= +github.com/uptrace/bun/driver/pgdriver v1.1.12/go.mod h1:ssYUP+qwSEgeDDS1xm2XBip9el1y9Mi5mTAvLoiADLM= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.69.0 h1:fNLLESD2SooWeh2cidsuFtOcrEi4uB4m1mPrkJMZyVI= github.com/valyala/fasthttp v1.69.0/go.mod h1:4wA4PfAraPlAsJ5jMSqCE2ug5tqUPwKXxVj8oNECGcw= +github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94= +github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= +github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/wneessen/go-mail v0.7.2 h1:xxPnhZ6IZLSgxShebmZ6DPKh1b6OJcoHfzy7UjOkzS8= github.com/wneessen/go-mail v0.7.2/go.mod h1:+TkW6QP3EVkgTEqHtVmnAE/1MRhmzb8Y9/W3pweuS+k= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -224,6 +289,18 @@ github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZ github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +gitlab.com/golang-commonmark/html v0.0.0-20191124015941-a22733972181 h1:K+bMSIx9A7mLES1rtG+qKduLIXq40DAzYHtb0XuCukA= +gitlab.com/golang-commonmark/html v0.0.0-20191124015941-a22733972181/go.mod h1:dzYhVIwWCtzPAa4QP98wfB9+mzt33MSmM8wsKiMi2ow= +gitlab.com/golang-commonmark/linkify v0.0.0-20191026162114-a0c2df6c8f82 h1:oYrL81N608MLZhma3ruL8qTM4xcpYECGut8KSxRY59g= +gitlab.com/golang-commonmark/linkify v0.0.0-20191026162114-a0c2df6c8f82/go.mod h1:Gn+LZmCrhPECMD3SOKlE+BOHwhOYD9j7WT9NUtkCrC8= +gitlab.com/golang-commonmark/markdown v0.0.0-20211110145824-bf3e522c626a h1:O85GKETcmnCNAfv4Aym9tepU8OE0NmcZNqPlXcsBKBs= +gitlab.com/golang-commonmark/markdown v0.0.0-20211110145824-bf3e522c626a/go.mod h1:LaSIs30YPGs1H5jwGgPhLzc8vkNc/k0rDX/fEZqiU/M= +gitlab.com/golang-commonmark/mdurl v0.0.0-20191124015652-932350d1cb84 h1:qqjvoVXdWIcZCLPMlzgA7P9FZWdPGPvP/l3ef8GzV6o= +gitlab.com/golang-commonmark/mdurl v0.0.0-20191124015652-932350d1cb84/go.mod h1:IJZ+fdMvbW2qW6htJx7sLJ04FEs4Ldl/MDsJtMKywfw= +gitlab.com/golang-commonmark/puny v0.0.0-20191124015043-9f83538fa04f h1:Wku8eEdeJqIOFHtrfkYUByc4bCaTeA6fL0UJgfEiFMI= +gitlab.com/golang-commonmark/puny v0.0.0-20191124015043-9f83538fa04f/go.mod h1:Tiuhl+njh/JIg0uS/sOJVYi0x2HEa5rc1OAaVsb5tAs= +gitlab.com/opennota/wd v0.0.0-20180912061657-c5d65f63c638 h1:uPZaMiz6Sz0PZs3IZJWpU5qHKGNy///1pacZC9txiUI= +gitlab.com/opennota/wd v0.0.0-20180912061657-c5d65f63c638/go.mod h1:EGRJaqe2eO9XGmFtQCvV3Lm9NLico3UhFwUpCG/+mVU= go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU= go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= @@ -261,16 +338,22 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/api v0.273.0 h1:r/Bcv36Xa/te1ugaN1kdJ5LoA5Wj/cL+a4gj6FiPBjQ= google.golang.org/api v0.273.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew= +google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE= +google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw= +google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js= +google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y= google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w= google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= @@ -285,6 +368,12 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.5.4 h1:Iyrp9Meh3GmbSuyIAGyjkN+n9K+GHX9b9MqsTL4EJCo= +gorm.io/driver/postgres v1.5.4/go.mod h1:Bgo89+h0CRcdA33Y6frlaHHVuTdOf87pmyzwW9C/BH0= +gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= +gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo= +mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= modernc.org/libc v1.37.6 h1:orZH3c5wmhIQFTXF+Nt+eeauyd+ZIt2BX6ARe+kD+aw= modernc.org/libc v1.37.6/go.mod h1:YAXkAZ8ktnkCKaN9sw/UDeUVkGYJ/YquGO4FTi5nmHE= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= @@ -293,3 +382,5 @@ modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= modernc.org/sqlite v1.28.0 h1:Zx+LyDDmXczNnEQdvPuEfcFVA2ZPyaD7UCZDjef3BHQ= modernc.org/sqlite v1.28.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/internal/controllers/chatbotController.go b/internal/controllers/chatbotController.go new file mode 100644 index 0000000..b20d71e --- /dev/null +++ b/internal/controllers/chatbotController.go @@ -0,0 +1,60 @@ +package controllers + +import ( + "context" + "history-api/internal/dtos/request" + "history-api/internal/dtos/response" + "history-api/internal/services" + "history-api/pkg/validator" + "time" + + "github.com/gofiber/fiber/v3" +) + +type ChatbotController struct { + chatbotService services.ChatbotService +} + +func NewChatbotController(chatbotService services.ChatbotService) *ChatbotController { + return &ChatbotController{ + chatbotService: chatbotService, + } +} + +// Chat godoc +// @Summary Ask the AI chatbot +// @Description Ask a history question based on project context or global knowledge using RAG +// @Tags Chatbot +// @Accept json +// @Produce json +// @Security BearerAuth +// @Param request body request.ChatbotDto true "Chatbot query" +// @Success 200 {object} response.CommonResponse{data=string} "Successful response with AI answer" +// @Failure 400 {object} response.CommonResponse "Invalid request" +// @Failure 500 {object} response.CommonResponse "Internal server error" +// @Router /chatbot/chat [post] +func (cx *ChatbotController) Chat(c fiber.Ctx) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + dto := &request.ChatbotDto{} + if err := validator.ValidateBodyDto(c, dto); err != nil { + return c.Status(fiber.StatusBadRequest).JSON(response.CommonResponse{ + Status: false, + Errors: err, + }) + } + + answer, err := cx.chatbotService.Chat(ctx, dto.ProjectID, dto.Question) + if err != nil { + return c.Status(fiber.StatusInternalServerError).JSON(response.CommonResponse{ + Status: false, + Message: err.Error(), + }) + } + + return c.Status(fiber.StatusOK).JSON(response.CommonResponse{ + Status: true, + Data: answer, + }) +} diff --git a/internal/dtos/request/chatbot.go b/internal/dtos/request/chatbot.go new file mode 100644 index 0000000..aa3a814 --- /dev/null +++ b/internal/dtos/request/chatbot.go @@ -0,0 +1,6 @@ +package request + +type ChatbotDto struct { + ProjectID *string `json:"project_id"` + Question string `json:"question" validate:"required"` +} diff --git a/internal/dtos/request/snapshot.go b/internal/dtos/request/snapshot.go index 662261c..b9d23d1 100644 --- a/internal/dtos/request/snapshot.go +++ b/internal/dtos/request/snapshot.go @@ -83,7 +83,7 @@ type WikiSnapshot struct { Source string `json:"source,omitempty" validate:"omitempty,oneof=inline ref"` Operation string `json:"operation,omitempty" validate:"omitempty,oneof=create update delete reference"` Title string `json:"title" validate:"required"` - Doc json.RawMessage `json:"doc,omitempty"` + Doc string `json:"doc,omitempty"` UpdatedAt string `json:"updated_at,omitempty"` } diff --git a/internal/dtos/response/wiki.go b/internal/dtos/response/wiki.go index f6dad46..0618b20 100644 --- a/internal/dtos/response/wiki.go +++ b/internal/dtos/response/wiki.go @@ -1,16 +1,15 @@ package response import ( - "encoding/json" "time" ) type WikiResponse struct { - ID string `json:"id"` - Title string `json:"title,omitempty"` - Content json.RawMessage `json:"content,omitempty"` - ProjectID string `json:"project_id"` - IsDeleted bool `json:"is_deleted,omitempty"` - CreatedAt *time.Time `json:"created_at,omitempty"` - UpdatedAt *time.Time `json:"updated_at,omitempty"` + ID string `json:"id"` + Title string `json:"title,omitempty"` + Content string `json:"content,omitempty"` + ProjectID string `json:"project_id"` + IsDeleted bool `json:"is_deleted,omitempty"` + CreatedAt *time.Time `json:"created_at,omitempty"` + UpdatedAt *time.Time `json:"updated_at,omitempty"` } diff --git a/internal/gen/sqlc/models.go b/internal/gen/sqlc/models.go index d3c7a65..5118547 100644 --- a/internal/gen/sqlc/models.go +++ b/internal/gen/sqlc/models.go @@ -8,6 +8,7 @@ import ( "encoding/json" "github.com/jackc/pgx/v5/pgtype" + "github.com/pgvector/pgvector-go" ) type Commit struct { @@ -94,6 +95,18 @@ type ProjectMember struct { CreatedAt pgtype.Timestamptz `json:"created_at"` } +type RagChunk struct { + ID pgtype.UUID `json:"id"` + SourceType string `json:"source_type"` + SourceID pgtype.UUID `json:"source_id"` + ProjectID pgtype.UUID `json:"project_id"` + ChunkIndex int32 `json:"chunk_index"` + Content string `json:"content"` + Embedding pgvector.Vector `json:"embedding"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + UpdatedAt pgtype.Timestamptz `json:"updated_at"` +} + type Role struct { ID pgtype.UUID `json:"id"` Name string `json:"name"` @@ -170,7 +183,7 @@ type Wiki struct { ID pgtype.UUID `json:"id"` ProjectID pgtype.UUID `json:"project_id"` Title pgtype.Text `json:"title"` - Content []byte `json:"content"` + Content pgtype.Text `json:"content"` IsDeleted bool `json:"is_deleted"` CreatedAt pgtype.Timestamptz `json:"created_at"` UpdatedAt pgtype.Timestamptz `json:"updated_at"` diff --git a/internal/gen/sqlc/rag.sql.go b/internal/gen/sqlc/rag.sql.go new file mode 100644 index 0000000..dcc6a48 --- /dev/null +++ b/internal/gen/sqlc/rag.sql.go @@ -0,0 +1,138 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: rag.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/pgvector/pgvector-go" +) + +const createRagChunk = `-- name: CreateRagChunk :one +INSERT INTO rag_chunks ( + id, source_type, source_id, project_id, chunk_index, content, embedding +) VALUES ( + COALESCE($7::uuid, uuidv7()), + $1, $2, $3, $4, $5, $6 +) +RETURNING id, source_type, source_id, project_id, chunk_index, content, embedding, created_at, updated_at +` + +type CreateRagChunkParams struct { + SourceType string `json:"source_type"` + SourceID pgtype.UUID `json:"source_id"` + ProjectID pgtype.UUID `json:"project_id"` + ChunkIndex int32 `json:"chunk_index"` + Content string `json:"content"` + Embedding pgvector.Vector `json:"embedding"` + ID pgtype.UUID `json:"id"` +} + +func (q *Queries) CreateRagChunk(ctx context.Context, arg CreateRagChunkParams) (RagChunk, error) { + row := q.db.QueryRow(ctx, createRagChunk, + arg.SourceType, + arg.SourceID, + arg.ProjectID, + arg.ChunkIndex, + arg.Content, + arg.Embedding, + arg.ID, + ) + var i RagChunk + err := row.Scan( + &i.ID, + &i.SourceType, + &i.SourceID, + &i.ProjectID, + &i.ChunkIndex, + &i.Content, + &i.Embedding, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const deleteRagChunksBySourceIDs = `-- name: DeleteRagChunksBySourceIDs :exec +DELETE FROM rag_chunks +WHERE source_type = $1 AND source_id = ANY($2::uuid[]) +` + +type DeleteRagChunksBySourceIDsParams struct { + SourceType string `json:"source_type"` + Column2 []pgtype.UUID `json:"column_2"` +} + +func (q *Queries) DeleteRagChunksBySourceIDs(ctx context.Context, arg DeleteRagChunksBySourceIDsParams) error { + _, err := q.db.Exec(ctx, deleteRagChunksBySourceIDs, arg.SourceType, arg.Column2) + return err +} + +const searchRagChunks = `-- name: SearchRagChunks :many +SELECT + id, source_type, source_id, project_id, chunk_index, content, + (1 - (embedding <=> $1))::float8 AS similarity +FROM rag_chunks +WHERE 1=1 + AND ($2::uuid IS NULL OR project_id = $2::uuid) + AND ($3::varchar IS NULL OR source_type = $3::varchar) + AND (1 - (embedding <=> $1))::float8 >= $4::float8 +ORDER BY embedding <=> $1 +LIMIT $5 +` + +type SearchRagChunksParams struct { + Embedding pgvector.Vector `json:"embedding"` + ProjectID pgtype.UUID `json:"project_id"` + SourceType pgtype.Text `json:"source_type"` + MatchThreshold float64 `json:"match_threshold"` + MatchCount int32 `json:"match_count"` +} + +type SearchRagChunksRow struct { + ID pgtype.UUID `json:"id"` + SourceType string `json:"source_type"` + SourceID pgtype.UUID `json:"source_id"` + ProjectID pgtype.UUID `json:"project_id"` + ChunkIndex int32 `json:"chunk_index"` + Content string `json:"content"` + Similarity float64 `json:"similarity"` +} + +func (q *Queries) SearchRagChunks(ctx context.Context, arg SearchRagChunksParams) ([]SearchRagChunksRow, error) { + rows, err := q.db.Query(ctx, searchRagChunks, + arg.Embedding, + arg.ProjectID, + arg.SourceType, + arg.MatchThreshold, + arg.MatchCount, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []SearchRagChunksRow{} + for rows.Next() { + var i SearchRagChunksRow + if err := rows.Scan( + &i.ID, + &i.SourceType, + &i.SourceID, + &i.ProjectID, + &i.ChunkIndex, + &i.Content, + &i.Similarity, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/gen/sqlc/wiki.sql.go b/internal/gen/sqlc/wiki.sql.go index f4a8241..3601679 100644 --- a/internal/gen/sqlc/wiki.sql.go +++ b/internal/gen/sqlc/wiki.sql.go @@ -77,7 +77,7 @@ RETURNING id, project_id, title, content, is_deleted, created_at, updated_at type CreateWikiParams struct { Title pgtype.Text `json:"title"` - Content []byte `json:"content"` + Content pgtype.Text `json:"content"` ProjectID pgtype.UUID `json:"project_id"` ID pgtype.UUID `json:"id"` } @@ -312,7 +312,7 @@ RETURNING id, project_id, title, content, is_deleted, created_at, updated_at type UpdateWikiParams struct { Title pgtype.Text `json:"title"` - Content []byte `json:"content"` + Content pgtype.Text `json:"content"` ProjectID pgtype.UUID `json:"project_id"` ID pgtype.UUID `json:"id"` } diff --git a/internal/models/rag.go b/internal/models/rag.go new file mode 100644 index 0000000..5401b74 --- /dev/null +++ b/internal/models/rag.go @@ -0,0 +1,17 @@ +package models + +import ( + "time" +) + +type RagChunk struct { + ID string `json:"id"` + SourceType string `json:"source_type"` + SourceID string `json:"source_id"` + ProjectID string `json:"project_id"` + ChunkIndex int32 `json:"chunk_index"` + Content string `json:"content"` + Similarity float64 `json:"similarity,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} diff --git a/internal/models/wiki.go b/internal/models/wiki.go index e023fe5..2595306 100644 --- a/internal/models/wiki.go +++ b/internal/models/wiki.go @@ -1,19 +1,18 @@ package models import ( - "encoding/json" "history-api/internal/dtos/response" "time" ) type WikiEntity struct { - ID string `json:"id"` - Title string `json:"title"` - Content json.RawMessage `json:"content"` - ProjectID string `json:"project_id"` - IsDeleted bool `json:"is_deleted"` - CreatedAt *time.Time `json:"created_at"` - UpdatedAt *time.Time `json:"updated_at"` + ID string `json:"id"` + Title string `json:"title"` + Content string `json:"content"` + ProjectID string `json:"project_id"` + IsDeleted bool `json:"is_deleted"` + CreatedAt *time.Time `json:"created_at"` + UpdatedAt *time.Time `json:"updated_at"` } func (w *WikiEntity) ToResponse() *response.WikiResponse { diff --git a/internal/repositories/ragRepository.go b/internal/repositories/ragRepository.go new file mode 100644 index 0000000..259ff2a --- /dev/null +++ b/internal/repositories/ragRepository.go @@ -0,0 +1,94 @@ +package repositories + +import ( + "context" + "history-api/internal/gen/sqlc" + "history-api/internal/models" + "history-api/pkg/cache" + "history-api/pkg/convert" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "github.com/pgvector/pgvector-go" +) + +type RagRepository interface { + SaveChunk(ctx context.Context, sourceType string, sourceID string, projectID string, index int, content string, vector []float32) error + SearchSimilar(ctx context.Context, projectID *string, vector []float32, limit int, threshold float64) ([]*models.RagChunk, error) + DeleteBySourceIDs(ctx context.Context, sourceType string, sourceIDs []string) error + WithTx(tx pgx.Tx) RagRepository +} + +type ragRepository struct { + q *sqlc.Queries + c cache.Cache +} + +func NewRagRepository(db sqlc.DBTX, c cache.Cache) RagRepository { + return &ragRepository{q: sqlc.New(db), c: c} +} + +func (r *ragRepository) WithTx(tx pgx.Tx) RagRepository { + return &ragRepository{q: r.q.WithTx(tx), c: r.c} +} + +func (r *ragRepository) SaveChunk(ctx context.Context, sourceType string, sourceID string, projectID string, index int, content string, vector []float32) error { + pID, _ := convert.StringToUUID(projectID) + sID, _ := convert.StringToUUID(sourceID) + + _, err := r.q.CreateRagChunk(ctx, sqlc.CreateRagChunkParams{ + SourceType: sourceType, + SourceID: sID, + ProjectID: pID, + ChunkIndex: int32(index), + Content: content, + Embedding: pgvector.NewVector(vector), + }) + return err +} + +func (r *ragRepository) SearchSimilar(ctx context.Context, projectID *string, vector []float32, limit int, threshold float64) ([]*models.RagChunk, error) { + params := sqlc.SearchRagChunksParams{ + Embedding: pgvector.NewVector(vector), + MatchThreshold: threshold, + MatchCount: int32(limit), + } + if projectID != nil && *projectID != "" { + pID, _ := convert.StringToUUID(*projectID) + params.ProjectID = pID + } + + rows, err := r.q.SearchRagChunks(ctx, params) + if err != nil { + return nil, err + } + + res := make([]*models.RagChunk, len(rows)) + for i, row := range rows { + res[i] = &models.RagChunk{ + ID: convert.UUIDToString(row.ID), + Content: row.Content, + Similarity: row.Similarity, + } + } + return res, nil +} + +func (r *ragRepository) DeleteBySourceIDs(ctx context.Context, sourceType string, sourceIDs []string) error { + if len(sourceIDs) == 0 { + return nil + } + + uids := make([]pgtype.UUID, 0, len(sourceIDs)) + for _, id := range sourceIDs { + uid, err := convert.StringToUUID(id) + if err == nil { + uids = append(uids, uid) + } + } + + return r.q.DeleteRagChunksBySourceIDs(ctx, sqlc.DeleteRagChunksBySourceIDsParams{ + SourceType: sourceType, + Column2: uids, + }) +} diff --git a/internal/repositories/wikiRepository.go b/internal/repositories/wikiRepository.go index 81d2ee0..6aaab9b 100644 --- a/internal/repositories/wikiRepository.go +++ b/internal/repositories/wikiRepository.go @@ -90,7 +90,7 @@ func (r *wikiRepository) getByIDsWithFallback(ctx context.Context, ids []string) item := models.WikiEntity{ ID: convert.UUIDToString(row.ID), Title: convert.TextToString(row.Title), - Content: json.RawMessage(row.Content), + Content: convert.TextToString(row.Content), IsDeleted: row.IsDeleted, ProjectID: convert.UUIDToString(row.ProjectID), CreatedAt: convert.TimeToPtr(row.CreatedAt), @@ -143,7 +143,7 @@ func (r *wikiRepository) GetByID(ctx context.Context, id pgtype.UUID) (*models.W wiki = models.WikiEntity{ ID: convert.UUIDToString(row.ID), Title: convert.TextToString(row.Title), - Content: json.RawMessage(row.Content), + Content: convert.TextToString(row.Content), IsDeleted: row.IsDeleted, CreatedAt: convert.TimeToPtr(row.CreatedAt), UpdatedAt: convert.TimeToPtr(row.UpdatedAt), @@ -172,7 +172,7 @@ func (r *wikiRepository) Search(ctx context.Context, params sqlc.SearchWikisPara wiki := &models.WikiEntity{ ID: convert.UUIDToString(row.ID), Title: convert.TextToString(row.Title), - Content: json.RawMessage(row.Content), + Content: convert.TextToString(row.Content), IsDeleted: row.IsDeleted, CreatedAt: convert.TimeToPtr(row.CreatedAt), UpdatedAt: convert.TimeToPtr(row.UpdatedAt), @@ -201,7 +201,7 @@ func (r *wikiRepository) Create(ctx context.Context, params sqlc.CreateWikiParam wiki := models.WikiEntity{ ID: convert.UUIDToString(row.ID), Title: convert.TextToString(row.Title), - Content: json.RawMessage(row.Content), + Content: convert.TextToString(row.Content), IsDeleted: row.IsDeleted, CreatedAt: convert.TimeToPtr(row.CreatedAt), UpdatedAt: convert.TimeToPtr(row.UpdatedAt), @@ -218,7 +218,7 @@ func (r *wikiRepository) Update(ctx context.Context, params sqlc.UpdateWikiParam wiki := models.WikiEntity{ ID: convert.UUIDToString(row.ID), Title: convert.TextToString(row.Title), - Content: json.RawMessage(row.Content), + Content: convert.TextToString(row.Content), IsDeleted: row.IsDeleted, CreatedAt: convert.TimeToPtr(row.CreatedAt), UpdatedAt: convert.TimeToPtr(row.UpdatedAt), @@ -272,7 +272,7 @@ func (r *wikiRepository) GetByProjectID(ctx context.Context, projectID pgtype.UU wiki := &models.WikiEntity{ ID: convert.UUIDToString(row.ID), Title: convert.TextToString(row.Title), - Content: json.RawMessage(row.Content), + Content: convert.TextToString(row.Content), IsDeleted: row.IsDeleted, ProjectID: convert.UUIDToString(row.ProjectID), CreatedAt: convert.TimeToPtr(row.CreatedAt), diff --git a/internal/routes/chatbotRoute.go b/internal/routes/chatbotRoute.go new file mode 100644 index 0000000..3288c5d --- /dev/null +++ b/internal/routes/chatbotRoute.go @@ -0,0 +1,16 @@ +package routes + +import ( + "history-api/internal/controllers" + "history-api/internal/middlewares" + "history-api/internal/repositories" + + "github.com/gofiber/fiber/v3" +) + +func ChatbotRoutes(app *fiber.App, controller *controllers.ChatbotController, userRepo repositories.UserRepository) { + route := app.Group("/chatbot") + + route.Use(middlewares.JwtAccess(userRepo)) + route.Post("/chat", controller.Chat) +} diff --git a/internal/services/chatbotService.go b/internal/services/chatbotService.go new file mode 100644 index 0000000..a0731bc --- /dev/null +++ b/internal/services/chatbotService.go @@ -0,0 +1,51 @@ +package services + +import ( + "context" + "fmt" + "history-api/internal/repositories" + "history-api/pkg/ai" +) + +type ChatbotService interface { + Chat(ctx context.Context, projectID *string, question string) (string, error) +} + +type chatbotService struct { + repo repositories.RagRepository + ragUtils *ai.RagUtils +} + +func NewChatbotService(repo repositories.RagRepository, ragUtils *ai.RagUtils) ChatbotService { + return &chatbotService{ + repo: repo, + ragUtils: ragUtils, + } +} + +func (s *chatbotService) Chat(ctx context.Context, projectID *string, question string) (string, error) { + qVector, err := s.ragUtils.EmbedQuery(ctx, question) + if err != nil { + return "", fmt.Errorf("failed to embed question: %w", err) + } + results, err := s.repo.SearchSimilar(ctx, projectID, qVector, 5, 0.65) + if err != nil { + return "", fmt.Errorf("failed to search similar content: %w", err) + } + + contextStr := "" + for i, res := range results { + contextStr += fmt.Sprintf("[Document %d]: %s\n", i+1, res.Content) + } + + prompt := fmt.Sprintf(`You are a helpful history assistant. Answer the question based ONLY on the provided context. +If the answer is not in the context, say "I don't have enough historical context to answer that." + +Context: +%s + +Question: %s +Answer:`, contextStr, question) + + return s.ragUtils.GenerateResponse(ctx, prompt) +} diff --git a/internal/services/submissionService.go b/internal/services/submissionService.go index ec3ac09..777f961 100644 --- a/internal/services/submissionService.go +++ b/internal/services/submissionService.go @@ -9,6 +9,7 @@ import ( "history-api/internal/gen/sqlc" "history-api/internal/models" "history-api/internal/repositories" + "history-api/pkg/ai" "history-api/pkg/cache" "history-api/pkg/constants" "history-api/pkg/convert" @@ -36,6 +37,8 @@ type submissionService struct { wikiRepo repositories.WikiRepository geometryRepo repositories.GeometryRepository entityRepo repositories.EntityRepository + ragRepo repositories.RagRepository + ragUtils *ai.RagUtils db *pgxpool.Pool c cache.Cache } @@ -48,6 +51,8 @@ func NewSubmissionService( wikiRepo repositories.WikiRepository, geometryRepo repositories.GeometryRepository, entityRepo repositories.EntityRepository, + ragRepo repositories.RagRepository, + ragUtils *ai.RagUtils, db *pgxpool.Pool, c cache.Cache, ) SubmissionService { @@ -59,6 +64,8 @@ func NewSubmissionService( wikiRepo: wikiRepo, geometryRepo: geometryRepo, entityRepo: entityRepo, + ragRepo: ragRepo, + ragUtils: ragUtils, db: db, c: c, } @@ -127,6 +134,7 @@ func (s *submissionService) UpdateSubmissionStatus(ctx context.Context, reviewer entityRepo := s.entityRepo.WithTx(tx) geometryRepo := s.geometryRepo.WithTx(tx) wikiRepo := s.wikiRepo.WithTx(tx) + ragRepo := s.ragRepo.WithTx(tx) submissionUUID, err := convert.StringToUUID(submissionID) if err != nil { @@ -166,8 +174,11 @@ func (s *submissionService) UpdateSubmissionStatus(ctx context.Context, reviewer return nil, fiber.NewError(fiber.StatusBadRequest, "Commit does not belong to project") } + listDeleteEntities := make([]pgtype.UUID, 0) + listDeleteWikis := make([]pgtype.UUID, 0) + listDeleteGeometries := make([]pgtype.UUID, 0) + var snapshotData request.CommitSnapshot if status == constants.StatusTypeApproved { - var snapshotData request.CommitSnapshot err = json.Unmarshal(commit.SnapshotJson, &snapshotData) if err != nil { return nil, fiber.NewError(fiber.StatusInternalServerError, "Failed to parse commit snapshot") @@ -214,7 +225,6 @@ func (s *submissionService) UpdateSubmissionStatus(ctx context.Context, reviewer persistCurrentItemIDs[item.ID] = struct{}{} } - listDeleteEntities := make([]pgtype.UUID, 0) for _, e := range currentEntity { if _, ok := persistItemIDs[e.ID]; !ok { itemUUID, err := convert.StringToUUID(e.ID) @@ -226,7 +236,6 @@ func (s *submissionService) UpdateSubmissionStatus(ctx context.Context, reviewer } } - listDeleteGeometries := make([]pgtype.UUID, 0) for _, g := range currentGeometry { if _, ok := persistItemIDs[g.ID]; !ok { itemUUID, err := convert.StringToUUID(g.ID) @@ -238,7 +247,6 @@ func (s *submissionService) UpdateSubmissionStatus(ctx context.Context, reviewer } } - listDeleteWikis := make([]pgtype.UUID, 0) for _, w := range currentWiki { if _, ok := persistItemIDs[w.ID]; !ok { itemUUID, err := convert.StringToUUID(w.ID) @@ -274,6 +282,7 @@ func (s *submissionService) UpdateSubmissionStatus(ctx context.Context, reviewer refEntityIDs = append(refEntityIDs, e.ID) } } + refEntities, _ := s.entityRepo.GetByIDs(ctx, refEntityIDs) refEntityMap := make(map[string]bool) for _, e := range refEntities { @@ -444,7 +453,7 @@ func (s *submissionService) UpdateSubmissionStatus(ctx context.Context, reviewer _, err := wikiRepo.Update(ctx, sqlc.UpdateWikiParams{ ID: wikiUUID, Title: convert.StringToText(wiki.Title), - Content: wiki.Doc, + Content: convert.StringToText(wiki.Doc), ProjectID: projectUUID, }) if err != nil { @@ -456,7 +465,7 @@ func (s *submissionService) UpdateSubmissionStatus(ctx context.Context, reviewer _, err := wikiRepo.Create(ctx, sqlc.CreateWikiParams{ ID: wikiUUID, Title: convert.StringToText(wiki.Title), - Content: wiki.Doc, + Content: convert.StringToText(wiki.Doc), ProjectID: projectUUID, }) if err != nil { @@ -554,6 +563,58 @@ func (s *submissionService) UpdateSubmissionStatus(ctx context.Context, reviewer } } + if status == constants.StatusTypeApproved { + wikiDeleteIDs := make([]string, 0) + entityDeleteIDs := make([]string, 0) + + for _, id := range listDeleteWikis { + wikiDeleteIDs = append(wikiDeleteIDs, convert.UUIDToString(id)) + } + for _, id := range listDeleteEntities { + entityDeleteIDs = append(entityDeleteIDs, convert.UUIDToString(id)) + } + + for _, wiki := range snapshotData.Wikis { + if wiki.Operation == "delete" { + wikiDeleteIDs = append(wikiDeleteIDs, wiki.ID) + } + } + for _, entity := range snapshotData.Entities { + if entity.Operation == "delete" { + entityDeleteIDs = append(entityDeleteIDs, entity.ID) + } + } + + _ = ragRepo.DeleteBySourceIDs(ctx, "wiki", wikiDeleteIDs) + _ = ragRepo.DeleteBySourceIDs(ctx, "entity", entityDeleteIDs) + + for _, wiki := range snapshotData.Wikis { + if wiki.Source == "inline" { + cleanText := s.ragUtils.StripHTML(wiki.Title + "\n" + wiki.Doc) + chunks, vectors, err := s.ragUtils.PrepareChunks(ctx, cleanText) + if err == nil { + _ = ragRepo.DeleteBySourceIDs(ctx, "wiki", []string{wiki.ID}) + for i, chunk := range chunks { + _ = ragRepo.SaveChunk(ctx, "wiki", wiki.ID, commit.ProjectID, i, chunk, vectors[i]) + } + } + } + } + + for _, entity := range snapshotData.Entities { + if entity.Source == "inline" { + cleanText := s.ragUtils.StripHTML(entity.Name + "\n" + entity.Description) + chunks, vectors, err := s.ragUtils.PrepareChunks(ctx, cleanText) + if err == nil { + _ = ragRepo.DeleteBySourceIDs(ctx, "entity", []string{entity.ID}) + for i, chunk := range chunks { + _ = ragRepo.SaveChunk(ctx, "entity", entity.ID, commit.ProjectID, i, chunk, vectors[i]) + } + } + } + } + } + arg := sqlc.UpdateSubmissionParams{ ID: submissionUUID, Status: pgtype.Int2{Int16: status.Int16(), Valid: true}, @@ -563,12 +624,12 @@ func (s *submissionService) UpdateSubmissionStatus(ctx context.Context, reviewer updatedSubmission, err := submissionRepo.Update(ctx, arg) if err != nil { - return nil, fiber.NewError(fiber.StatusInternalServerError, "Failed to update submission status: " + err.Error()) + return nil, fiber.NewError(fiber.StatusInternalServerError, "Failed to update submission status: "+err.Error()) } err = tx.Commit(ctx) if err != nil { - return nil, fiber.NewError(fiber.StatusInternalServerError, "Failed to commit transaction: " + err.Error()) + return nil, fiber.NewError(fiber.StatusInternalServerError, "Failed to commit transaction: "+err.Error()) } if status == constants.StatusTypeApproved { @@ -579,6 +640,7 @@ func (s *submissionService) UpdateSubmissionStatus(ctx context.Context, reviewer _ = s.c.DelByPattern(bgCtx, "wiki:search*") }() } + _ = s.c.Del(ctx, fmt.Sprintf("project:id:%s", submission.ProjectID)) return updatedSubmission.ToResponse(), nil diff --git a/pkg/ai/rag.go b/pkg/ai/rag.go new file mode 100644 index 0000000..01d8a74 --- /dev/null +++ b/pkg/ai/rag.go @@ -0,0 +1,81 @@ +package ai + +import ( + "context" + "fmt" + "history-api/pkg/config" + "html" + "regexp" + + "github.com/tmc/langchaingo/embeddings" + "github.com/tmc/langchaingo/llms" + "github.com/tmc/langchaingo/llms/googleai" + "github.com/tmc/langchaingo/textsplitter" +) + +type RagUtils struct { + llm llms.Model + embedder *embeddings.EmbedderImpl +} + +func NewRagUtils() (*RagUtils, error) { + googleAIApiKey, err := config.GetConfig("GOOGLE_AI_API_KEY") + if err != nil { + return nil, err + } + + llm, err := googleai.New(context.Background(), + googleai.WithAPIKey(googleAIApiKey), + googleai.WithDefaultEmbeddingModel("gemini-embedding-001"), + ) + if err != nil { + return nil, fmt.Errorf("failed to init google ai: %w", err) + } + + embedder, err := embeddings.NewEmbedder(llm) + if err != nil { + return nil, fmt.Errorf("failed to init embedder: %w", err) + } + + return &RagUtils{ + llm: llm, + embedder: embedder, + }, nil +} + +func (u *RagUtils) StripHTML(text string) string { + re := regexp.MustCompile(`<[^>]*>`) + text = re.ReplaceAllString(text, " ") + return html.UnescapeString(text) +} + +func (u *RagUtils) PrepareChunks(ctx context.Context, text string) ([]string, [][]float32, error) { + splitter := textsplitter.NewRecursiveCharacter( + textsplitter.WithChunkSize(1000), + textsplitter.WithChunkOverlap(200), + ) + + chunks, err := splitter.SplitText(text) + if err != nil || len(chunks) == 0 { + return nil, nil, err + } + + vectors, err := u.embedder.EmbedDocuments(ctx, chunks) + if err != nil { + return nil, nil, err + } + + return chunks, vectors, nil +} + +func (u *RagUtils) EmbedQuery(ctx context.Context, query string) ([]float32, error) { + vectors, err := u.embedder.EmbedDocuments(ctx, []string{query}) + if err != nil || len(vectors) == 0 { + return nil, err + } + return vectors[0], nil +} + +func (u *RagUtils) GenerateResponse(ctx context.Context, prompt string) (string, error) { + return llms.GenerateFromSinglePrompt(ctx, u.llm, prompt) +} diff --git a/sqlc.yaml b/sqlc.yaml index a0794e3..f40b016 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -21,4 +21,8 @@ sql: - db_type: json go_type: encoding/json.RawMessage - db_type: jsonb - go_type: encoding/json.RawMessage \ No newline at end of file + go_type: encoding/json.RawMessage + - db_type: vector + go_type: + import: "github.com/pgvector/pgvector-go" + type: "Vector" \ No newline at end of file