Go
Building a basic RAG Agent with GoodMem
View source on Github Run in Codespace
Building a Basic RAG Agent with GoodMem (Go)
Overview
This tutorial will guide you through building a complete Retrieval-Augmented Generation (RAG) system using GoodMem's vector memory capabilities with the Go SDK. By the end of this guide, you'll have a functional Q&A system that can:
- 🔍 Semantically search through your documents
- 📝 Generate contextual answers using retrieved information
- 🏗️ Scale to handle large document collections
What is RAG?
RAG combines the power of retrieval (finding relevant information) with generation (creating natural language responses). This approach allows AI systems to provide accurate, context-aware answers by:
- Retrieving relevant documents from a knowledge base
- Augmenting the query with this context
- Generating a comprehensive answer using both the query and retrieved information
Why GoodMem for RAG?
GoodMem provides enterprise-grade vector storage with:
- Multiple embedder support for optimal retrieval accuracy
- Streaming APIs for real-time responses
- Advanced post-processing with reranking and summarization
- Scalable architecture for production workloads
Prerequisites
Before starting, ensure you have:
- ✅ GoodMem server running (install with:
curl -s https://get.goodmem.ai | bash) - ✅ Go 1.18+ installed
- ✅ API key for your GoodMem instance
Installation & Setup
First, let's set up the Go module and install the required packages:
import (
"context"
"fmt"
"log"
"os"
"time"
"github.com/janpfeifer/gonb/cache" // Used by gonb to persist variables across cells
goodmem_client "github.com/PAIR-Systems-Inc/goodmem/clients/go"
)
// Helper functions for pointer creation
func PtrInt32(v int32) *int32 { return &v }
func PtrBool(v bool) *bool { return &v }
func PtrString(v string) *string { return &v }
func PtrFloat64(v float64) *float64 { return &v }Authentication & Configuration
Let's configure our GoodMem client and test the connection:
// Configuration - Update these values for your setup
var (
GOODMEM_HOST = getEnv("GOODMEM_HOST", "localhost:8080")
GOODMEM_API_KEY = getEnv("GOODMEM_API_KEY", "your-api-key-here")
)
func getEnv(key, defaultValue string) string {
if value := os.Getenv(key); value != "" {
return value
}
return defaultValue
}
%%
fmt.Printf("GoodMem Host: %s\n", GOODMEM_HOST)
if GOODMEM_API_KEY == "your-api-key-here" {
fmt.Println("API Key configured: No - Please update")
} else {
fmt.Println("API Key configured: Yes")
}// Create GoodMem API client
func getClient() *goodmem_client.APIClient {
configuration := goodmem_client.NewConfiguration()
configuration.Host = GOODMEM_HOST
configuration.Scheme = "http"
configuration.DefaultHeader["X-API-Key"] = GOODMEM_API_KEY
client := goodmem_client.NewAPIClient(configuration)
return client
}
%%
client := getClient()
ctx := context.Background()
// Test connection by listing spaces
listResponse, httpResp, err := client.SpacesAPI.ListSpaces(ctx).Execute()
if err != nil {
log.Fatalf("❌ Error connecting to GoodMem: %v (HTTP Status: %d)", err, httpResp.StatusCode)
}
fmt.Println("✅ Successfully connected to GoodMem!")
if listResponse.Spaces != nil {
fmt.Printf(" Found %d existing spaces\n", len(listResponse.Spaces))
}Creating Your First Space
In GoodMem, a Space is a logical container for organizing memories. Each space has:
- Associated embedders for generating vector representations
- Access controls (public/private)
- Metadata labels for organization
Let's create a space for our RAG demo:
// First, let's see what embedders are available
func getEmbedders() []goodmem_client.EmbedderResponse {
client := getClient()
ctx := context.Background()
listResponse, httpResp, err := client.EmbeddersAPI.ListEmbedders(ctx).Execute()
if err != nil {
log.Fatalf("❌ Error connecting to GoodMem: %v (HTTP Status: %d)", err, httpResp.StatusCode)
}
return listResponse.Embedders
}
%%
availableEmbedders := getEmbedders()
fmt.Printf("📋 Available Embedders (%d):\n", len(availableEmbedders))
for i, embedder := range availableEmbedders {
fmt.Printf(" %d. %s - %s\n", i+1, embedder.DisplayName, embedder.ProviderType)
fmt.Printf(" Model: %s\n", embedder.ModelIdentifier)
fmt.Printf(" ID: %s\n", embedder.EmbedderId)
fmt.Println()
}
var defaultEmbedder *goodmem_client.EmbedderResponse
if len(availableEmbedders) > 0 {
defaultEmbedder = &availableEmbedders[0]
fmt.Printf("🎯 Using embedder: %s\n", defaultEmbedder.DisplayName)
} else {
fmt.Println("⚠️ No embedders found. You may need to configure an embedder first.")
fmt.Println(" Refer to the documentation: https://docs.goodmem.ai/docs/reference/cli/goodmem_embedder_create/")
}// Execute to clear gonb cache on demoSpaceId
%%
cache.ResetKey("demoSpaceId")// Create a space for our RAG demo
const SPACE_NAME = "RAG Demo Knowledge Base (Go)"
// Define chunking configuration that we'll reuse throughout the tutorial
func get_chunking_config() *goodmem_client.ChunkingConfiguration {
jsonData := `
{
"recursive": {
"chunkSize": 256,
"chunkOverlap": 25,
"separators": ["\n\n", "\n", ". ", " ", ""],
"keepStrategy": "KEEP_END",
"separatorIsRegex": false,
"lengthMeasurement": "CHARACTER_COUNT"
}
}`
var config goodmem_client.NullableChunkingConfiguration
json.Unmarshal([]byte(jsonData), &config)
return config.Get()
}
var DEMO_CHUNKING_CONFIG = get_chunking_config()
func create_demo_space() string {
client := getClient()
ctx := context.Background()
// Check if space already exists
existingSpaces, _, _ := client.SpacesAPI.ListSpaces(ctx).Execute()
var demoSpace *goodmem_client.Space
for _, space := range existingSpaces.Spaces {
if space.Name == SPACE_NAME {
fmt.Printf("📁 Space '%s' already exists\n", SPACE_NAME)
fmt.Printf(" Space ID: %s\n", space.SpaceId)
fmt.Println(" To remove existing space, see https://docs.goodmem.ai/docs/reference/cli/goodmem_space_delete/")
demoSpace = &space
return demoSpace.SpaceId
}
}
if demoSpace == nil {
// Configure space embedders if we have available embedders
defaultEmbedder := getEmbedders()[0]
var spaceEmbedders []goodmem_client.SpaceEmbedderConfig
spaceEmbedders = []goodmem_client.SpaceEmbedderConfig{
{
EmbedderId: defaultEmbedder.EmbedderId,
DefaultRetrievalWeight: 1.0,
},
}
falseValue := false
falseBool := goodmem_client.NewNullableBool(&falseValue)
// Create space request
createRequest := goodmem_client.SpaceCreationRequest{
Name: SPACE_NAME,
Labels: map[string]string{
"purpose": "rag-demo",
"environment": "tutorial",
"content-type": "documentation",
},
SpaceEmbedders: spaceEmbedders,
PublicRead: *falseBool,
DefaultChunkingConfig: DEMO_CHUNKING_CONFIG,
}
// Create the space
newSpace, httpResp, err := client.SpacesAPI.CreateSpace(ctx).SpaceCreationRequest(createRequest).Execute()
if err != nil {
log.Fatalf("❌ Error creating space: %v (HTTP Status: %d)", err, httpResp.StatusCode)
}
demoSpace = newSpace
fmt.Printf("✅ Created space: %s\n", newSpace.Name)
fmt.Printf(" Space ID: %s\n", newSpace.SpaceId)
fmt.Printf(" Embedders: %d\n", len(newSpace.SpaceEmbedders))
if newSpace.Labels != nil {
fmt.Printf(" Labels: %v\n", newSpace.Labels)
}
fmt.Println(" Chunking Config Saved: 256 chars with 25 overlap")
fmt.Println(" 💡 This chunking config will be reused for all memory creation!")
return demoSpace.SpaceId
}
return ""
}
var demoSpaceId = cache.Cache("demoSpaceId", create_demo_space)// Verify our space configuration
%%
if demoSpaceId != "" {
client := getClient()
ctx := context.Background()
spaceDetails, httpResp, err := client.SpacesAPI.GetSpace(ctx, demoSpaceId).Execute()
if err != nil {
log.Fatalf("❌ Error getting space details: %v (HTTP Status: %d)", err, httpResp.StatusCode)
}
fmt.Println("🔍 Space Configuration:")
fmt.Printf(" Name: %s\n", spaceDetails.Name)
fmt.Printf(" Owner ID: %s\n", spaceDetails.OwnerId)
fmt.Printf(" Public Read: %v\n", spaceDetails.PublicRead)
fmt.Printf(" Created: %d\n", spaceDetails.CreatedAt)
if spaceDetails.Labels != nil {
fmt.Printf(" Labels: %v\n", spaceDetails.Labels)
}
fmt.Println("\n🤖 Associated Embedders:")
for _, embedderAssoc := range spaceDetails.SpaceEmbedders {
fmt.Printf(" Embedder ID: %s\n", embedderAssoc.EmbedderId)
fmt.Printf(" Retrieval Weight: %.1f\n", embedderAssoc.DefaultRetrievalWeight)
}
} else {
fmt.Println("⚠️ No space available for the demo")
}Adding Documents to Memory
Now let's add some sample documents to our space. GoodMem will automatically:
- Chunk the documents into optimal sizes
- Generate embeddings using the configured embedders
- Index the content for fast retrieval
We'll use sample company documents that represent common business use cases:
import (
"io/ioutil"
"path/filepath"
)
// Document structure
type Document struct {
Filename string
Description string
Content string
}
// Load sample documents
func loadSampleDocuments() []Document {
documents := []Document{}
sampleDir := "sample_documents"
docFiles := map[string]string{
"company_handbook.txt": "Employee handbook with policies and procedures",
"technical_documentation.txt": "API documentation and technical guides",
"product_faq.txt": "Frequently asked questions about products",
"security_policy.txt": "Information security policies and procedures",
}
for filename, description := range docFiles {
filepath := filepath.Join(sampleDir, filename)
content, err := ioutil.ReadFile(filepath)
if err != nil {
fmt.Printf("⚠️ File not found: %s\n", filepath)
continue
}
documents = append(documents, Document{
Filename: filename,
Description: description,
Content: string(content),
})
fmt.Printf("📄 Loaded: %s (%d characters)\n", filename, len(content))
}
return documents
}
// Load the documents
var sampleDocs = cache.Cache("sampleDocs", loadSampleDocuments)
%%
fmt.Printf("\n📚 Total documents loaded: %d\n", len(sampleDocs))// Execute to clear gonb cache on memoryId
%%
cache.ResetKey("memoryId")import "strings"
// Create the first memory individually to demonstrate single memory creation
func createMemory() string {
createSingleMemory := func(spaceId string, document Document) (*goodmem_client.Memory, error) {
// Extract document type from filename
docType := strings.Split(document.Filename, "_")[0]
// Create memory request
memoryRequest := goodmem_client.MemoryCreationRequest{
SpaceId: spaceId,
OriginalContent: *goodmem_client.NewNullableString(&document.Content),
ContentType: "text/plain",
Metadata: map[string]interface{}{
"filename": document.Filename,
"description": document.Description,
"source": "sample_documents",
"document_type": docType,
"ingestion_method": "single",
},
ChunkingConfig: DEMO_CHUNKING_CONFIG,
}
// Create the memory
client := getClient()
ctx := context.Background()
memory, httpResp, err := client.MemoriesAPI.CreateMemory(ctx).MemoryCreationRequest(memoryRequest).Execute()
if err != nil {
return nil, fmt.Errorf("failed to create memory: %v (HTTP Status: %d)", err, httpResp.StatusCode)
}
fmt.Printf("✅ Created single memory: %s\n", document.Filename)
fmt.Printf(" Memory ID: %s\n", memory.MemoryId)
fmt.Printf(" Status: %s\n", memory.ProcessingStatus)
fmt.Printf(" Content Length: %d characters\n", len(document.Content))
fmt.Println()
return memory, nil
}
var singleMemory *goodmem_client.Memory
if len(sampleDocs) > 0 {
firstDoc := sampleDocs[0]
fmt.Println("📝 Creating first document using CreateMemory API:")
fmt.Printf(" Document: %s\n", firstDoc.Filename)
fmt.Println(" Method: Individual memory creation")
fmt.Println()
memory, err := createSingleMemory(demoSpaceId, firstDoc)
if err != nil {
fmt.Printf("⚠️ Single memory creation failed: %v\n", err)
} else {
singleMemory = memory
fmt.Println("🎯 Single memory creation completed successfully!")
}
} else {
fmt.Println("⚠️ Cannot create memory: missing space or documents")
}
return singleMemory.MemoryId
}
var memoryId = cache.Cache("memoryId", createMemory)import "encoding/base64"
%%
// Demonstrate retrieving a memory by ID using get_memory
fmt.Println("📖 Retrieving memory details using GetMemory API:")
fmt.Printf(" Memory ID: %s\n", memoryId)
fmt.Println()
client := getClient()
ctx := context.Background()
// Retrieve the memory without content
retrievedMemory, httpResp, err := client.MemoriesAPI.GetMemory(ctx, memoryId).IncludeContent(false).Execute()
if err != nil {
log.Fatalf("❌ Error retrieving memory: %v (HTTP Status: %d)", err, httpResp.StatusCode)
}
fmt.Println("✅ Successfully retrieved memory:")
fmt.Printf(" Memory ID: %s\n", retrievedMemory.MemoryId)
fmt.Printf(" Space ID: %s\n", retrievedMemory.SpaceId)
fmt.Printf(" Status: %s\n", retrievedMemory.ProcessingStatus)
fmt.Printf(" Content Type: %s\n", retrievedMemory.ContentType)
fmt.Printf(" Created At: %d\n", retrievedMemory.CreatedAt)
fmt.Printf(" Updated At: %d\n", retrievedMemory.UpdatedAt)
if retrievedMemory.Metadata != nil {
fmt.Println("\n 📋 Metadata:")
for key, value := range retrievedMemory.Metadata {
fmt.Printf(" %s: %v\n", key, value)
}
}
// Now retrieve with content included
fmt.Println("\n📖 Retrieving memory with content:")
retrievedWithContent, httpResp, err := client.MemoriesAPI.GetMemory(ctx, memoryId).IncludeContent(true).Execute()
if err != nil {
log.Fatalf("❌ Error retrieving memory with content: %v (HTTP Status: %d)", err, httpResp.StatusCode)
}
if retrievedWithContent.OriginalContent.IsSet() {
// Decode the base64 encoded content
decodedContent, err := base64.StdEncoding.DecodeString(*retrievedWithContent.OriginalContent.Get())
if err != nil {
log.Fatalf("❌ Error decoding content: %v", err)
}
contentStr := string(decodedContent)
fmt.Println("✅ Content retrieved and decoded:")
fmt.Printf(" Content length: %d characters\n", len(contentStr))
if len(contentStr) > 200 {
fmt.Printf(" First 200 chars: %s...\n", contentStr[:200])
} else {
fmt.Printf(" Content: %s\n", contentStr)
}
} else {
fmt.Println("⚠️ No content available")
}// Create the remaining documents using batch memory creation
func createBatchMemories(spaceId string, documents []Document) error {
var memoryRequests []goodmem_client.MemoryCreationRequest
for _, doc := range documents {
docType := strings.Split(doc.Filename, "_")[0]
memoryRequest := goodmem_client.MemoryCreationRequest{
SpaceId: spaceId,
OriginalContent: *goodmem_client.NewNullableString(&doc.Content),
ContentType: "text/plain",
ChunkingConfig: DEMO_CHUNKING_CONFIG,
Metadata: map[string]interface{}{
"filename": doc.Filename,
"description": doc.Description,
"source": "sample_documents",
"document_type": docType,
"ingestion_method": "batch",
},
}
memoryRequests = append(memoryRequests, memoryRequest)
}
// Create batch request
batchRequest := goodmem_client.BatchMemoryCreationRequest{
Requests: memoryRequests,
}
fmt.Printf("📦 Creating %d memories using BatchCreateMemory API:\n", len(memoryRequests))
client := getClient()
ctx := context.Background()
// Execute batch creation
httpResp, err := client.MemoriesAPI.BatchCreateMemory(ctx).BatchMemoryCreationRequest(batchRequest).Execute()
if err != nil {
return fmt.Errorf("batch creation failed: %v (HTTP Status: %d)", err, httpResp.StatusCode)
}
return nil
}
%%
if len(sampleDocs) > 1 {
// Create the remaining documents (skip the first one we already created)
remainingDocs := sampleDocs[1:]
err := createBatchMemories(demoSpaceId, remainingDocs)
if err != nil {
fmt.Printf("⚠️ Batch creation error: %v\n", err)
}
fmt.Println("\n📋 Total Memory Creation Summary:")
fmt.Println(" 📄 Single CreateMemory: 1 document")
fmt.Printf(" 📦 Batch CreateMemory: %d documents submitted\n", len(remainingDocs))
fmt.Println(" ⏳ Check processing status in the next cell")
} else {
fmt.Println("⚠️ Cannot create batch memories: insufficient documents or missing space")
}// List all memories in our space to verify they're ready
%%
client := getClient()
ctx := context.Background()
memoriesResponse, httpResp, err := client.MemoriesAPI.ListMemories(ctx, demoSpaceId).Execute()
if err != nil {
log.Fatalf("❌ Failed to list memories: %v (HTTP Status: %d)", err, httpResp.StatusCode)
}
memories := memoriesResponse.Memories
fmt.Printf("📚 Memories in space '%s':\n", demoSpaceId)
fmt.Printf(" Total memories: %d\n", len(memories))
fmt.Println()
for i, memory := range memories {
var filename, description string
if memory.Metadata != nil {
if fn, ok := (memory.Metadata)["filename"]; ok {
filename = fmt.Sprintf("%v", fn)
} else {
filename = "Unknown"
}
if desc, ok := (memory.Metadata)["description"]; ok {
description = fmt.Sprintf("%v", desc)
} else {
description = "No description"
}
}
fmt.Printf(" %d. %s\n", i+1, filename)
fmt.Printf(" Status: %s\n", memory.ProcessingStatus)
fmt.Printf(" Description: %s\n", description)
fmt.Printf(" Created: %d\n", memory.CreatedAt)
fmt.Println()
}// Monitor processing status for all created memories
func waitForProcessingCompletion(spaceId string, maxWaitSeconds int) bool {
fmt.Println("⏳ Waiting for document processing to complete...")
fmt.Println(" 💡 Note: Batch memories are processed asynchronously, so we check by listing all memories in the space")
fmt.Println()
startTime := time.Now()
maxWaitDuration := time.Duration(maxWaitSeconds) * time.Second
client := getClient()
ctx := context.Background()
for time.Since(startTime) < maxWaitDuration {
memoriesResponse, httpResp, err := client.MemoriesAPI.ListMemories(ctx, spaceId).Execute()
if err != nil {
fmt.Printf("❌ Error checking processing status: %v (HTTP Status: %d)\n", err, httpResp.StatusCode)
return false
}
memories := memoriesResponse.Memories
// Check processing status
statusCounts := make(map[string]int)
for _, memory := range memories {
statusCounts[memory.ProcessingStatus]++
}
fmt.Printf("📊 Processing status: %v (Total: %d memories)\n", statusCounts, len(memories))
// Check if all are completed
allCompleted := true
for _, memory := range memories {
if memory.ProcessingStatus != "COMPLETED" {
allCompleted = false
break
}
}
if allCompleted {
fmt.Println("✅ All documents processed successfully!")
return true
}
// Check for failures
if failedCount, ok := statusCounts["FAILED"]; ok && failedCount > 0 {
fmt.Printf("❌ %d memories failed processing\n", failedCount)
return false
}
time.Sleep(5 * time.Second)
}
fmt.Printf("⏰ Timeout waiting for processing (waited %ds)\n", maxWaitSeconds)
return false
}
%%
processingComplete := waitForProcessingCompletion(demoSpaceId, 120)
if processingComplete {
fmt.Println("🎉 Ready for semantic search and retrieval!")
fmt.Println("📈 Batch API benefit: Multiple documents submitted in a single API call")
fmt.Println("🔧 Consistent chunking: All memories use DEMO_CHUNKING_CONFIG")
} else {
fmt.Println("⚠️ Some documents may still be processing. You can continue with the tutorial.")
}Semantic Search & Retrieval
Now comes the exciting part! Let's perform semantic search using GoodMem's streaming API. This will:
- Find relevant chunks based on semantic similarity
- Stream results in real-time
- Include relevance scores for ranking
- Return structured data for easy processing
// ChunkResult represents a search result chunk
type ChunkResult struct {
ChunkText string
RelevanceScore float64
MemoryIndex int32
ResultSetID string
ChunkSequence int32
}
// Perform semantic search using GoodMem's streaming API
func semanticSearch(query string, spaceId string, maxResults int32) []ChunkResult {
fmt.Printf("🔍 Searching for: '%s'\n", query)
fmt.Printf("📁 Space ID: %s\n", spaceId)
fmt.Printf("📊 Max results: %d\n", maxResults)
fmt.Println(strings.Repeat("-", 50))
client := getClient()
ctx := context.Background()
// Create streaming client
streamingClient := goodmem_client.NewStreamingClient(client)
// Create stream request
streamReq := &goodmem_client.MemoryStreamRequest{
Message: query,
SpaceIDs: []string{spaceId},
RequestedSize: PtrInt32(maxResults),
FetchMemory: PtrBool(true),
FetchMemoryContent: PtrBool(false),
GenerateAbstract: PtrBool(false),
Format: goodmem_client.FormatNDJSON,
}
// Perform streaming search
streamCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
stream, err := streamingClient.RetrieveMemoryStream(streamCtx, streamReq)
if err != nil {
fmt.Printf("❌ Failed to start streaming: %v\n", err)
return nil
}
eventCount := 0
var retrievedChunks []ChunkResult
for event := range stream {
eventCount++
if event.RetrievedItem != nil && event.RetrievedItem.Chunk != nil {
chunkInfo := event.RetrievedItem.Chunk
chunkData := chunkInfo.Chunk
var chunkText string
var chunkSeq int32
if text, ok := chunkData["chunkText"]; ok {
chunkText = fmt.Sprintf("%v", text)
}
if seq, ok := chunkData["chunkSequenceNumber"]; ok {
if seqFloat, ok := seq.(float64); ok {
chunkSeq = int32(seqFloat)
}
}
result := ChunkResult{
ChunkText: chunkText,
RelevanceScore: chunkInfo.RelevanceScore,
MemoryIndex: int32(chunkInfo.MemoryIndex),
ResultSetID: chunkInfo.ResultSetId,
ChunkSequence: chunkSeq,
}
retrievedChunks = append(retrievedChunks, result)
fmt.Printf("📄 Chunk %d:\n", len(retrievedChunks))
fmt.Printf(" Relevance: %.3f\n", chunkInfo.RelevanceScore)
displayText := chunkText
if len(displayText) > 200 {
displayText = displayText[:200] + "..."
}
fmt.Printf(" Text: %s\n", displayText)
fmt.Println()
}
}
fmt.Printf("✅ Search completed: %d chunks found, %d events processed\n", len(retrievedChunks), eventCount)
return retrievedChunks
}
%%
// Test semantic search with a sample query
sampleQuery := "What is the vacation policy for employees?"
semanticSearch(sampleQuery, demoSpaceId, 5)// Let's try a few different queries to see how semantic search works
func testMultipleQueries(spaceId string) {
testQueries := []string{
"How do I reset my password?",
"What are the security requirements for remote work?",
"API authentication and rate limits",
"Employee benefits and health insurance",
"How much does the software cost?",
}
for i, query := range testQueries {
fmt.Printf("\n🔍 Test Query %d: %s\n", i+1, query)
fmt.Println(strings.Repeat("=", 60))
semanticSearch(query, spaceId, 3)
fmt.Println("\n" + strings.Repeat("-", 60))
}
}
%%
testMultipleQueries(demoSpaceId)Next Steps & Advanced Features
Congratulations! 🎉 You've successfully built a semantic search system using GoodMem with Go. Here's what you've accomplished:
✅ What You Built
- Document ingestion pipeline with automatic chunking and embedding
- Semantic search system with relevance scoring
- Simple Q&A system using GoodMem's vector capabilities
🚀 Next Steps for Advanced Implementation
1. Multiple Embedders & Reranking
- Coming Soon
2. Integration with Popular Frameworks
- Coming Soon
3. Advanced Post-Processing
- Coming Soon
📚 Additional Resources
GoodMem Documentation: