[Notebooks] cleanup (#2947)

* update readme & error 500

Signed-off-by: Lucas <lyoon@redhat.com>

* update prompting, fixing documentation

Signed-off-by: Lucas <lyoon@redhat.com>

---------

Signed-off-by: Lucas <lyoon@redhat.com>

Author: JslYoon

workspaces/lightspeed/.changeset/selfish-trainers-play.md

@@ -0,0 +1,5 @@
+---
+'@red-hat-developer-hub/backstage-plugin-lightspeed-backend': minor
+---
+
+updated config.d.ts to reflect app-config.yaml notebooks settings. Update notebooks system prompting

workspaces/lightspeed/plugins/lightspeed-backend/README.md

@@ -31,7 +31,7 @@ Add the following lightspeed configurations into your `app-config.yaml` file:
 
 ```yaml
 lightspeed:
-  servicePort: <portNumber> # Optional - Change the LS service port nubmer. Defaults to 8080.
+  servicePort: <portNumber> # Optional - Change the LS service port number. Defaults to 8080.
   systemPrompt: <system prompt> # Optional - Override the default system prompt.
   mcpServers: # Optional - one or more MCP servers
     - name: <mcp server name> # must match the name configured in LCS
@@ -129,7 +129,7 @@ lightspeed:
 
 **Notebooks Settings**:
 
-- **`Notebooks.enabled`** _(optional)_: Enable or disable the Notebooks feature (default: `false`)
+- **`notebooks.enabled`** _(optional)_: Enable or disable the Notebooks feature (default: `false`)
 
 **Query Defaults** _(required when enabled)_:
 

workspaces/lightspeed/plugins/lightspeed-backend/config.d.ts

@@ -52,69 +52,48 @@ export interface Config {
     /**
      * Configuration for AI Notebooks (Developer Preview)
      */
-    aiNotebooks?: {
+    notebooks?: {
       /**
        * Enable/disable AI Notebooks feature
        * When enabled, exposes AI Notebooks REST API endpoints for document-based conversations with RAG.
-       * Requires Llama Stack service to be running (default: http://0.0.0.0:8321).
+       * Requires Lightspeed service to be running (default: http://0.0.0.0:8080).
        * @default false
        * @visibility frontend
        */
-      enabled?: boolean;
+      enabled: boolean;
       /**
-       * Llama Stack configuration
+       * Lightspeed configuration
        * @visibility backend
        */
-      llamaStack?: {
+      queryDefaults: {
         /**
-         * Llama Stack API port
+         * Model to use for answering queries. Must map to a model available through the provider set in provider_id.
          * @visibility backend
          */
-        port?: number;
+        model: string;
         /**
-         * Embedding model for vector database
+         * AI provider for the query model. Must map to a provider enabled in your Lightspeed config.
          * @visibility backend
          */
-        embeddingModel?: string;
-        /**
-         * Embedding dimension
-         * @visibility backend
-         */
-        embeddingDimension?: number;
-        /**
-         * Vector IO configuration
-         * @visibility backend
-         */
-        vectorIo?: {
-          /**
-           * Vector store provider ID
-           * @visibility backend
-           */
-          providerId?: string;
-        };
+        provider_id: string;
       };
       /**
-       * File processing timeout in milliseconds
-       * @visibility backend
-       */
-      fileProcessingTimeoutMs?: number;
-      /**
-       * Chunking strategy configuration
+       * Chunking strategy for document processing
        * @visibility backend
        */
       chunkingStrategy?: {
         /**
-         * Type of chunking strategy ('auto' or 'static')
+         * Document chunking strategy - 'auto' (automatic, default) or 'static' (fixed size)
          * @visibility backend
          */
-        type?: string;
+        type?: 'auto' | 'static';
         /**
-         * Maximum chunk size in tokens (for static strategy)
+         * Maximum chunk size in tokens for static chunking (default: 512)
          * @visibility backend
          */
         maxChunkSizeTokens?: number;
         /**
-         * Chunk overlap in tokens (for static strategy)
+         * Token overlap between chunks for static chunking (default: 50)
          * @visibility backend
          */
         chunkOverlapTokens?: number;

workspaces/lightspeed/plugins/lightspeed-backend/src/service/constant.ts

@@ -25,23 +25,47 @@ export const DEFAULT_LLAMA_STACK_PORT = 8321; // Llama Stack port
 export const DEFAULT_LIGHTSPEED_SERVICE_HOST = '0.0.0.0'; // Lightspeed core service host
 export const DEFAULT_LIGHTSPEED_SERVICE_PORT = 8080; // Lightspeed service port
 export const DEFAULT_MAX_FILE_SIZE_MB = 20 * 1024 * 1024; // 20MB
-export const NOTEBOOKS_SYSTEM_PROMPT =
-  `You are an expert Research Analyst. Your goal is to synthesize information across provided documents to answer user queries with high precision.
-
-Constraints:
-- Groundedness: Only use information explicitly stated in or directly inferred from the documents. If the answer isn't present, state: "I don't know based on the provided documents."
-- Citations: Every claim must be followed by an inline citation (e.g., [Document Title/Id]).
-- Tone: Maintain a professional, objective, and analytical tone.
-- Conflicting Info: If documents contradict each other, highlight the discrepancy rather than choosing one.
-
-Output Format:
-1. Summary: A 1-2 sentence high-level answer.
-2. Detailed Analysis: A structured breakdown using bullet points.
-3. References: A list of sources used. References should be in the format of [Document Title] in a new line for each reference.
-
-Disclaimer: Your answers **MUST** be grounded in the provided documents. If the answer isn't present, state: "I don't know based on the provided documents."
-Remember, **ALL** references must be from the provided documents and provided documents only.
-Make no mistakes.
+export const NOTEBOOKS_SYSTEM_PROMPT = `
+You are a helpful, analytical Senior Research Analyst assistant. Your primary objective is to synthesize cross-document information to answer user queries with 100% fidelity to the provided documents.
+
+### QUERY TYPES - IMPORTANT
+* **Meta Queries ONLY:** ONLY when the user asks specifically about YOU as an assistant (e.g., "who are you", "what can you do", "hello"), respond naturally without requiring documents.
+* **ALL OTHER QUERIES:** For ANY other question, you MUST use the strict document-grounding rules below. This includes general knowledge questions, trivia, explanations, etc. If it's not about you as an assistant, it requires document evidence.
+
+### STRICT OPERATIONAL CONSTRAINTS
+* **Zero Outside Knowledge:** Do NOT use prior training data, general knowledge, or unsupported logical leaps to answer queries.
+* **Absolute Grounding:** If the provided documents do not contain explicit, direct evidence to answer the query, you MUST output exactly: "I cannot answer this based on the provided documents."
+* **Precision Citations:** Every single factual claim, metric, or conclusion must have an inline citation [Document Title].
+* **Contradictions:** Do not resolve discrepancies. If sources conflict, explicitly document the friction (e.g., "Source A states X, whereas Source B states Y").
+
+### ANALYTICAL GUIDELINES
+* **Comprehensive Responses:** Provide thorough, detailed analysis. Don't be overly brief - expand on the evidence with full context and explanation.
+* **Quantitative Focus:** Prioritize extracting specific metrics, dates, and figures.
+* **Objective Tone:** Use neutral, professional language. Do not use subjective adjectives (e.g., "impressive," "concerning") unless quoting the text directly.
+
+### REQUIRED ANALYSIS PROCESS
+Before generating your response, you must internally perform evidence extraction (DO NOT show this in your output):
+1. Identify the core entities and requirements of the user's query.
+2. Extract exact, verbatim quotes from the provided documents that directly address the query.
+3. If no explicit quotes exist to answer the prompt, output ONLY: "I cannot answer this based on the provided documents."
+
+### CRITICAL: NEVER output <evidence_extraction> tags or any internal reasoning in your visible response. These are for your internal analysis only.
+
+### REQUIRED OUTPUT STRUCTURE
+When you have evidence from documents, structure your response as:
+
+**Executive Summary:**
+[A comprehensive 2-4 sentence synthesis of the primary findings based strictly on the extracted evidence. Provide full context and detail.]
+
+**Detailed Analysis:**
+* **[Key Entity/Theme]:** [Thorough explanation of the fact or data point derived from text, with full context and supporting details] [Document Title].
+* **[Key Entity/Theme]:** [Thorough explanation of the fact or data point derived from text, with full context and supporting details] [Document Title].
+
+**Referenced Documents:**
+* [Document Title 1]
+* [Document Title 2]
+
+When you lack evidence, output ONLY: "I cannot answer this based on the provided documents."
 `.trim();
 
 /**

workspaces/lightspeed/plugins/lightspeed-backend/src/service/notebooks/VectorStoresOperator.ts

@@ -74,7 +74,16 @@ async function handleHttpError(
     error = { detail: await response.text() };
   }
   logger.error(`Failed to ${operation}:`, error);
-  throw mapHttpStatusToError(response.status, `Failed to ${operation}`, error);
+
+  let status = response.status;
+  if (status === 500 && operation === 'retrieve vector store') {
+    logger.warn(
+      `Treating 500 error as 404 for ${operation} (lightspeed-core limitation)`,
+    );
+    status = 404;
+  }
+
+  throw mapHttpStatusToError(status, `Failed to ${operation}`, error);
 }
 
 /**
@@ -171,7 +180,6 @@ export class VectorStoresOperator {
           },
         },
       );
-
       if (!response.ok) {
         await handleHttpError(response, this.logger, 'retrieve vector store');
       }

workspaces/lightspeed/plugins/lightspeed-backend/src/service/notebooks/notebooksRouter.test.ts

@@ -155,7 +155,43 @@ describe('Notebooks Router', () => {
       });
     });
 
+    describe('GET /v1/sessions/:sessionId', () => {
+      it('should return 404 for non-existing session', async () => {
+        const response = await request(app).get(
+          '/notebooks/v1/sessions/non-existing-session-id',
+        );
+
+        expect(response.status).toBe(404);
+        expect(response.body.status).toBe('error');
+      });
+
+      it('should retrieve an existing session', async () => {
+        const createResponse = await request(app)
+          .post('/notebooks/v1/sessions')
+          .send({ name: 'Test Session' });
+
+        const sessionId = createResponse.body.session.session_id;
+        const response = await request(app).get(
+          `/notebooks/v1/sessions/${sessionId}`,
+        );
+
+        expect(response.status).toBe(200);
+        expect(response.body.status).toBe('success');
+        expect(response.body.session.session_id).toBe(sessionId);
+        expect(response.body.session.name).toBe('Test Session');
+      });
+    });
+
     describe('PUT /v1/sessions/:sessionId', () => {
+      it('should return 404 for non-existing session', async () => {
+        const response = await request(app)
+          .put('/notebooks/v1/sessions/non-existing-session-id')
+          .send({ name: 'Updated Name' });
+
+        expect(response.status).toBe(404);
+        expect(response.body.status).toBe('error');
+      });
+
       it('should update session', async () => {
         const createResponse = await request(app)
           .post('/notebooks/v1/sessions')
@@ -172,6 +208,15 @@ describe('Notebooks Router', () => {
     });
 
     describe('DELETE /v1/sessions/:sessionId', () => {
+      it('should return 404 for non-existing session', async () => {
+        const response = await request(app).delete(
+          '/notebooks/v1/sessions/non-existing-session-id',
+        );
+
+        expect(response.status).toBe(404);
+        expect(response.body.status).toBe('error');
+      });
+
       it('should delete session', async () => {
         const createResponse = await request(app)
           .post('/notebooks/v1/sessions')
@@ -299,6 +344,15 @@ describe('Notebooks Router', () => {
       sessionId = response.body.session.session_id;
     });
 
+    it('should return 404 for non-existing session', async () => {
+      const response = await request(app)
+        .post('/notebooks/v1/sessions/non-existing-session/query')
+        .send({ query: 'What is this about?' });
+
+      expect(response.status).toBe(404);
+      expect(response.body.status).toBe('error');
+    });
+
     it('should return 400 if query missing', async () => {
       const response = await request(app)
         .post(`/notebooks/v1/sessions/${sessionId}/query`)

workspaces/lightspeed/plugins/lightspeed-backend/src/service/notebooks/notebooksRouters.ts

@@ -77,8 +77,10 @@ export async function createNotebooksRouter(
   );
   const systemPrompt = NOTEBOOKS_SYSTEM_PROMPT;
 
-  if ((queryModel && !queryProvider) || (!queryModel && queryProvider)) {
-    throw new Error('Query model and provider must be configured together');
+  if (!queryModel || !queryProvider) {
+    throw new Error(
+      'Query model and provider are required. Please configure lightspeed.notebooks.queryDefaults.model and lightspeed.notebooks.queryDefaults.provider_id',
+    );
   }
 
   logger.info(
@@ -231,15 +233,51 @@ export async function createNotebooksRouter(
             this.push(`data: ${JSON.stringify(legacy)}\n\n`);
           } else if (eventType === 'response.completed') {
             const usage = parsed?.response?.usage;
+
+            // Log the full response to see what we're getting
+            logger.info(
+              `Full response.completed event: ${JSON.stringify(parsed?.response, null, 2)}`,
+            );
+
+            // Extract citations/sources from tool calls (file_search results)
+            const toolCalls = parsed?.response?.tool_calls || [];
+            logger.info(
+              `Tool calls received: ${JSON.stringify(toolCalls, null, 2)}`,
+            );
+
+            const referencedDocuments: any[] = [];
+
+            for (const toolCall of toolCalls) {
+              if (toolCall.tool_name === 'file_search') {
+                logger.info(
+                  `Found file_search tool call: ${JSON.stringify(toolCall, null, 2)}`,
+                );
+                const citations = toolCall.content?.citations || [];
+                for (const citation of citations) {
+                  referencedDocuments.push({
+                    document_id: citation.document_id || citation.file_id,
+                    content: citation.text || citation.content,
+                  });
+                }
+              }
+            }
+
+            logger.info(
+              `Referenced documents: ${JSON.stringify(referencedDocuments)}`,
+            );
+
             const legacy = {
               event: 'end',
               data: {
-                referenced_documents: [],
+                referenced_documents: referencedDocuments,
                 input_tokens: usage?.input_tokens,
                 output_tokens: usage?.output_tokens,
               },
             };
             this.push(`data: ${JSON.stringify(legacy)}\n\n`);
+          } else {
+            // Log unhandled event types to help identify what we're missing
+            logger.debug(`Unhandled SSE event type: ${eventType}`, parsed);
           }
         }
 
@@ -458,6 +496,8 @@ export async function createNotebooksRouter(
         tools: [{ type: 'file_search', vector_store_ids: [sessionId] }],
         model: `${queryProvider}/${queryModel}`,
         stream: true,
+        temperature: 0.05,
+        shield_ids: [],
         max_tool_calls: 10,
         ...(conversationId && { conversation: conversationId }),
       };

workspaces/lightspeed/plugins/lightspeed/README.md

@@ -133,7 +133,7 @@ Notebooks is an experimental feature that enables **document-based conversations
 
 #### Prerequisites for Notebooks
 
-- Notebooks requires a **Llama Stack service** to be running
+- Notebooks requires a **Lightspeed Stack service** to be running
 - The backend administrator must enable the feature (see [Backend Configuration](../lightspeed-backend/README.md#notebooks-developer-preview))
 - Users need the appropriate RBAC permissions (if enabled)