feat: implement RAG chatbot service, background cron worker, and asynchronous indexing infrastructure

cmd/worker/cron/main.go

@@ -35,7 +35,6 @@ func runBackup(ctx context.Context, s3 storage.Storage, dbURI string) {
 	fileName := fmt.Sprintf("db_backup_%s.sql", time.Now().Format("2006-01-02_15-04-05"))
 	filePath := filepath.Join(tmpDir, fileName)
 
-	// Run pg_dump
 	cmd := exec.Command("pg_dump", dbURI, "-F", "c", "-f", filePath)
 	if err := cmd.Run(); err != nil {
 		log.Error().Err(err).Msg("Failed to execute pg_dump. Make sure pg_dump is installed.")

cmd/worker/rag/main.go

@@ -40,7 +40,6 @@ func processRagTask(ctx context.Context, ragRepo repositories.RagRepository, rag
 		}
 	}
 
-	// 2. Index wikis with delay + retry
 	for _, wiki := range task.Wikis {
 		if wiki.Source != "inline" {
 			continue
@@ -83,7 +82,6 @@ func processRagTask(ctx context.Context, ragRepo repositories.RagRepository, rag
 			continue
 		}
 
-		// Delete existing chunks then save new ones
 		_ = ragRepo.DeleteBySourceIDs(ctx, "wiki", []string{wiki.ID})
 		for i, chunk := range chunks {
 			if saveErr := ragRepo.SaveChunk(ctx, "wiki", wiki.ID, task.ProjectID, i, chunk, vectors[i]); saveErr != nil {
@@ -92,12 +90,9 @@ func processRagTask(ctx context.Context, ragRepo repositories.RagRepository, rag
 		}
 
 		log.Info().Str("worker", workerName).Str("wiki_id", wiki.ID).Int("chunks", len(chunks)).Msg("Wiki indexed successfully")
-
-		// Delay between items to avoid rate limit
 		time.Sleep(itemDelay)
 	}
 
-	// 3. Index entities with delay + retry
 	for _, entity := range task.Entities {
 		if entity.Source != "inline" {
 			continue
@@ -140,7 +135,6 @@ func processRagTask(ctx context.Context, ragRepo repositories.RagRepository, rag
 			continue
 		}
 
-		// Delete existing chunks then save new ones
 		_ = ragRepo.DeleteBySourceIDs(ctx, "entity", []string{entity.ID})
 		for i, chunk := range chunks {
 			if saveErr := ragRepo.SaveChunk(ctx, "entity", entity.ID, task.ProjectID, i, chunk, vectors[i]); saveErr != nil {