Skip to content

Feat/user summary prompt #444

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
astkwcz wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Feat/user summary prompt #444

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 33 additions & 21 deletions lib/TaskProcessing/SummaryProvider.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,11 @@ public function getOptionalInputShape(): array {
$this->l->t('The maximum number of words/tokens that can be generated in the completion.'),
EShapeType::Number
),
'system_prompt_summary' => new ShapeDescriptor(
$this->l->t('Summary system prompt'),
$this->l->t('Summary system prompt. When set, it overrides the default prompt and the Format and Complexity options are ignored.'),
EShapeType::Text
),
];
}

Expand All @@ -101,6 +106,7 @@ public function getOptionalInputShapeDefaults(): array {
'max_tokens' => $this->service->getMaxTokens(),
'format' => 'auto',
'complexity' => 'medium',
'system_prompt_summary' => '',
];
}

Expand Down Expand Up @@ -145,32 +151,38 @@ public function process(?string $userId, array $input, callable $reportProgress)

try {
$completions = [];

// Use the admin-configured prompt, then default. Format and complexity appended
$summarySystemPrompt = $this->service->getSystemPromptSummary();
if ($summarySystemPrompt === '') {
$summarySystemPrompt = self::DEFAULT_SYSTEM_PROMPT;
$customSystemPrompt = isset($input['system_prompt_summary']) && is_string($input['system_prompt_summary'])
? trim($input['system_prompt_summary'])
: '';
if ($customSystemPrompt !== '') {
// A user provided prompt overrides the others; format and complexity are ignored
$summarySystemPrompt = $customSystemPrompt . ' ';
} else {
$summarySystemPrompt .= ' ';
}
// Fallback to the admin-configured prompt, then to default. Format and complexity appended
$summarySystemPrompt = $this->service->getSystemPromptSummary();
if ($summarySystemPrompt === '') {
$summarySystemPrompt = self::DEFAULT_SYSTEM_PROMPT;
} else {
$summarySystemPrompt .= ' ';
}

if (isset($input['format'])) {
if ($input['format'] === 'paragraph') {
$summarySystemPrompt .= 'Return the summary as a paragraph. ';
} elseif ($input['format'] === 'bullet_points') {
$summarySystemPrompt .= 'Return the summary as a list of bullet points. ';
} elseif ($input['format'] === 'sentence') {
$summarySystemPrompt .= 'Return the summary as a single sentence. Do not include more than one sentence. ';
if (isset($input['format'])) {
if ($input['format'] === 'paragraph') {
$summarySystemPrompt .= 'Return the summary as a paragraph. ';
} elseif ($input['format'] === 'bullet_points') {
$summarySystemPrompt .= 'Return the summary as a list of bullet points. ';
} elseif ($input['format'] === 'sentence') {
$summarySystemPrompt .= 'Return the summary as a single sentence. Do not include more than one sentence. ';
}
}
}
if (isset($input['complexity'])) {
if ($input['complexity'] === 'complex') {
$summarySystemPrompt .= 'Use complex language and vocabulary appropriate for an expert in the subject. ';
} elseif ($input['complexity'] === 'simple') {
$summarySystemPrompt .= 'Use simple language and vocabulary appropriate for a 5 year old. ';
if (isset($input['complexity'])) {
if ($input['complexity'] === 'complex') {
$summarySystemPrompt .= 'Use complex language and vocabulary appropriate for an expert in the subject. ';
} elseif ($input['complexity'] === 'simple') {
$summarySystemPrompt .= 'Use simple language and vocabulary appropriate for a 5 year old. ';
}
}
}

if ($this->service->isUsingOpenAi() || $this->service->getChatEndpointEnabled()) {

foreach ($prompts as $p) {
Expand Down
2 changes: 1 addition & 1 deletion src/components/ServiceForm.vue
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@
:placeholder="t('integration_openai', 'Leave empty to use the default summary prompt')"
@update:model-value="onInput({ system_prompt_summary: $event })" />
<NcButton variant="tertiary"
:title="t('integration_openai', 'When set, it replaces default summary prompt. The Format and Complexity options are still appended to it.')">
:title="t('integration_openai', 'Used when a user does not provide their own summary prompt. The Format and Complexity options are still appended to it.')">
<template #icon>
<HelpCircleOutlineIcon />
</template>
Expand Down
82 changes: 82 additions & 0 deletions tests/unit/Providers/OpenAiProviderTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -677,6 +677,88 @@ public function testSummaryProviderWithAdminSystemPrompt(): void {
$this->quotaUsageMapper->deleteUserQuotaUsages(self::TEST_USER1);
}

public function testSummaryProviderWithUserSystemPrompt(): void {
// Define admin system prompt to ensure it is overwritten by user system prompt
$adminSystemPrompt = 'This is an admin system prompt for summarization.';
$service = $this->service->with(['system_prompt_summary' => $adminSystemPrompt]);

$summaryProvider = new SummaryProvider(
$this->openAiApiService,
$this->createMock(\OCP\IL10N::class),
$this->chunkService,
$service,
self::TEXT_MODEL,
);

// Ensure that complexity and format are ignored when user system prompt is set
$userSystemPrompt = 'This is a user system prompt for summarization.';
$complexity = 'complex';
$format = 'bullet_points';
$prompt = 'This is a test prompt';
$input = [
'input' => $prompt,
'system_prompt_summary' => $userSystemPrompt,
'format' => $format,
'complexity' => $complexity,
];

$n = 1;

$response = '{
"id": "chatcmpl-123",
"object": "chat.completion",
"created": 1677652288,
"model": "gpt-3.5-turbo-0613",
"system_fingerprint": "fp_44709d6fcb",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "This is a test response."
},
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 9,
"completion_tokens": 12,
"total_tokens": 21
}
}';

$url = self::OPENAI_API_BASE . 'chat/completions';

$options = ['timeout' => Application::OPENAI_DEFAULT_REQUEST_TIMEOUT, 'headers' => ['User-Agent' => Application::USER_AGENT, 'Authorization' => self::AUTHORIZATION_HEADER, 'Content-Type' => 'application/json']];
$options['body'] = json_encode([
'model' => Application::DEFAULT_COMPLETION_MODEL_ID,
'messages' => [
['role' => 'system', 'content' => $userSystemPrompt . ' '],
['role' => 'user', 'content' => $prompt],
],
'n' => $n,
'stream' => false,
'max_completion_tokens' => Application::DEFAULT_MAX_NUM_OF_TOKENS,
'user' => self::TEST_USER1,
]);

$iResponse = $this->createMock(\OCP\Http\Client\IResponse::class);
$iResponse->method('getBody')->willReturn($response);
$iResponse->method('getStatusCode')->willReturn(200);
$iResponse->method('getHeader')->with('Content-Type')->willReturn('application/json');

$this->iClient->expects($this->once())->method('post')->with($url, $options)->willReturn($iResponse);

$result = $summaryProvider->process(self::TEST_USER1, $input, fn () => true);
$this->assertEquals('This is a test response.', $result['output']);

// Check that token usage is logged properly
$usage = $this->quotaUsageMapper->getQuotaUnitsOfUser(self::TEST_USER1, Application::QUOTA_TYPE_TEXT);
$this->assertEquals(21, $usage);
// Clear quota usage
$this->quotaUsageMapper->deleteUserQuotaUsages(self::TEST_USER1);
}

public function testProofreadProvider(): void {
$proofreadProvider = new ProofreadProvider(
$this->openAiApiService,
Expand Down
Loading