diff --git a/application/single_app/config.py b/application/single_app/config.py index d28611fe9..606e97d33 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -101,7 +101,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.340" +VERSION = "0.261.341" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_diagram_operations.py b/application/single_app/functions_diagram_operations.py index 2b7a66094..675b92a74 100644 --- a/application/single_app/functions_diagram_operations.py +++ b/application/single_app/functions_diagram_operations.py @@ -86,6 +86,12 @@ def build_diagram_guidance_message(): - `classDiagram` for types, fields, and inheritance. - `gantt` for scheduled work, `mindmap` for hierarchies of ideas, `journey` for user journeys. +For a schedule or two-lane event timeline, use `flowchart TB` with a labelled subgraph for each +activity lane and `direction LR` inside each lane. Keep the supplied relative times in the +event labels (for example before, during, and after an activity); do not invent calendar dates, +measurements, or events to fill a lane. This shows event order, not a quantitatively scaled +time axis. Use a native data chart instead when the user needs plotted numerical values. + Keep the source valid so it renders on the first attempt: - Give every node a quoted label, for example `app["Simple Chat App Service"]`. Unquoted parentheses, braces, angle brackets, colons, `#`, and quotes inside a label break the parser. - Never use `end`, `graph`, `class`, `style`, `subgraph`, or `click` as a node id: they are reserved words and the diagram will not parse. Write `end_state` or `graph_node` instead. diff --git a/application/single_app/functions_orchestration_composition.py b/application/single_app/functions_orchestration_composition.py index 72eac712a..f364bc148 100644 --- a/application/single_app/functions_orchestration_composition.py +++ b/application/single_app/functions_orchestration_composition.py @@ -1,7 +1,7 @@ # functions_orchestration_composition.py -"""Explicit one-call content preparation from named authorized result readers. +"""Explicit content preparation from named authorized result readers. -Version: 0.261.305 +Version: 0.261.341 Refusals for a missing signed-in session reported as their own failure in: 0.261.209 Charts drawn upstream reach the model as their placement tokens in: 0.261.293 No retrieval, file-format inference, upload, publication, or implicit sibling inputs. @@ -12,9 +12,10 @@ proposal cards). In a turn that may propose a workflow they also receive the user's local time, worded as the workflow's scheduled runs will receive theirs. -JSON preparation states each declared output's exact shape, asks the endpoint for a JSON -object, and makes one corrective call when a reply breaks a declared rule. Neither reply is -logged; the log records only application codes and hashed identifiers. +JSON preparation states each declared output's exact shape and asks for a JSON object. +Preparation makes at most one corrective call for a broken output rule or an omitted +explicit chart/diagram. Continued visual omission keeps usable content for finalization's +partial-delivery report. Neither reply is logged; logs contain codes and hashed identifiers. When an input carries citation values, such as the Microsoft 365 sources an action step lists, the step is told to copy them verbatim after the claims they support (0.261.303). @@ -29,12 +30,13 @@ from content_screening.contracts import ScreeningError from functions_appinsights import log_event, workflow_log_context -from functions_chart_operations import normalize_inline_chart_markdown +from functions_chart_operations import INLINE_CHART_BLOCK_LANGUAGE, normalize_inline_chart_markdown +from functions_diagram_operations import MERMAID_BLOCK_LANGUAGE from functions_m365_citations import M365_ANSWER_STYLE_INSTRUCTIONS from functions_mixed_source_orchestration import MixedSourceCancellationError from functions_orchestration_context import conversation_reference_messages from functions_orchestration_deliverables import ( - compose_deliverable_guidance, place_deck_images, place_image_tokens, + compose_deliverable_guidance, missing_compose_visuals, place_deck_images, place_image_tokens, ) from functions_orchestration_memory import OrchestrationMemoryError from functions_orchestration_registry import ( @@ -199,6 +201,14 @@ def _compose_correction(error, outputs): )) +def _compose_visual_rules(step, values): + languages = {VISUAL_CHART: INLINE_CHART_BLOCK_LANGUAGE, VISUAL_DIAGRAM: MERMAID_BLOCK_LANGUAGE} + return ' '.join( + f'Include the explicitly requested {kind} in a ```{languages[kind]}``` block.' + for kind in missing_compose_visuals(step, values) + ) + + def validate_prepared_output(specification, value, *, profile_validator=None): """Validate public prepared data; profile validators are injected server code.""" kind = specification['kind'] @@ -328,10 +338,10 @@ def _carries_citation_values(inputs): def adapter_compose(step, context, *, settings, user_id, emit=None, cancel_requested=None): - """Prepare all declared outputs in one content-generation call. + """Prepare all declared outputs, with at most one corrective content-generation call. - A JSON reply that breaks a rule stated by the declared outputs gets exactly one - corrective call naming that rule. Any other failure, or a second bad reply, fails the step. + Output-contract errors and omitted explicit visuals share that allowance. An unusable + correction fails; usable content still missing its visual reaches final delivery checks. """ service = require_result_service(context) producer = context.result_producer(step) @@ -548,12 +558,25 @@ def prepared_values(response): recheck() response = invoke(messages, stage='orchestration_compose', metadata=metadata) recheck() + reply_error = None try: values = prepared_values(response) except ValueError as error: correction = None if plain_text else _compose_correction(error, outputs) if correction is None: raise + reply_error = error + correction = ' '.join(part for part in (correction, _compose_visual_rules(step, {})) if part) + else: + visual_rules = _compose_visual_rules(step, values) + correction = ' '.join(( + 'Your previous reply omitted an explicitly requested visual.', visual_rules, + 'Use only the supplied context; do not invent values or events to fill a visual.', + 'Return only the complete prepared text.' if plain_text else ' '.join(( + JSON_OUTPUT_POLICY, *(_output_shape(output) for output in outputs), + )), + )) if visual_rules else None + if correction: # One corrective call, told which declared rule the reply broke. The reply itself # stays in the model conversation only; it is never logged. retry_messages = [ @@ -565,24 +588,38 @@ def prepared_values(response): output_tokens=getattr(invoke, 'output_tokens', None), ) if retry_audit['decision'] != 'full_input': - raise - log_event( - '[ORCHESTRATION_EXECUTOR] Prepared content did not match its declared outputs; asking once more.', - level=logging.INFO, - extra={ - **workflow_log_context( - run_id=context.run_id, conversation_id=getattr(context, 'conversation_id', None), - step_id=step['step_id'], - ), - 'capability_id': step['capability_id'], 'reason': 'compose_output_retry', - 'error_type': type(error).__name__, - **({'execution_code': error.code} if isinstance(error, ResultContractError) else {}), - }, - ) - recheck() - response = invoke(retry_messages, stage='orchestration_compose', metadata=metadata) - recheck() - values = prepared_values(response) + if reply_error is not None: + raise reply_error + log_event( + '[ORCHESTRATION_EXECUTOR] Visual correction exceeds the context budget; retaining prepared content.', + level=logging.INFO, + extra={ + **workflow_log_context( + run_id=context.run_id, conversation_id=getattr(context, 'conversation_id', None), + step_id=step['step_id'], + ), + 'capability_id': step['capability_id'], 'reason': 'compose_visual_retry_budget', + }, + ) + else: + log_event( + '[ORCHESTRATION_EXECUTOR] Prepared content did not match its declared requirements; asking once more.', + level=logging.INFO, + extra={ + **workflow_log_context( + run_id=context.run_id, conversation_id=getattr(context, 'conversation_id', None), + step_id=step['step_id'], + ), + 'capability_id': step['capability_id'], + 'reason': 'compose_output_retry' if reply_error is not None else 'compose_visual_retry', + **({'error_type': type(reply_error).__name__} if reply_error is not None else {}), + **({'execution_code': reply_error.code} if isinstance(reply_error, ResultContractError) else {}), + }, + ) + recheck() + response = invoke(retry_messages, stage='orchestration_compose', metadata=metadata) + recheck() + values = prepared_values(response) partial = any(reader.completeness.status == 'partial' for reader in readers.values()) limitations = tuple(dict.fromkeys( limitation for reader in readers.values() for limitation in reader.completeness.limitations diff --git a/application/single_app/functions_orchestration_deliverables.py b/application/single_app/functions_orchestration_deliverables.py index 4dae4bbd2..9e4a82545 100644 --- a/application/single_app/functions_orchestration_deliverables.py +++ b/application/single_app/functions_orchestration_deliverables.py @@ -1,11 +1,12 @@ # functions_orchestration_deliverables.py """What the user asked to receive, and whether the plan can actually deliver each part. -Version: 0.261.293 +Version: 0.261.341 Implemented in: 0.261.138 Document merge recipes added in: 0.261.245 One-sheet or sheet-per-file question for spreadsheet merges added in: 0.261.246 Gathered results and charts must reach the answer or a file, with chart delivery notes, in: 0.261.293 +Explicit compose visual omission checks shared with final delivery in: 0.261.341 The planner lists its plan's deliverables first: the answer, files, images, charts, and diagrams the user asked for (``requested: explicit``) and anything it adds on its own @@ -1589,6 +1590,30 @@ def _draws_charts(step): ) +def has_visual_block(text, kind): + """The same format-presence check for compose correction and final answer delivery.""" + return _VISUAL_FENCES[kind] in text + + +def missing_compose_visuals(step, values): + """Explicit visuals this enabled compose step owes but its Markdown does not include. + + The compiled brief contains planned deliverables only. Unavailable, suggested and + separately produced images are not requirements for a corrective content call. + """ + if not step.get('enabled', True): + return [] + text = '\n'.join( + values[name] for name in _markdown_outputs(step) if type(values.get(name)) is str + ) + required = { + entry['kind'] for entry in step.get('deliverable_context') or () + if entry.get('relation') == 'delivers' and entry.get('requested') == REQUESTED_EXPLICIT + and entry.get('enabled', True) and entry.get('kind') in _VISUAL_FENCES + } + return [kind for kind in _VISUAL_FENCES if kind in required and not has_visual_block(text, kind)] + + def gathered_charts(plan, task_results, read_value): """The charts each charting gather step drew from its exact rows, by step id in plan order. @@ -1671,7 +1696,7 @@ def visual_delivery(plan, statuses, answer, charts): elif step['step_id'] in answer_steps: if not finished: reasons.append('step_not_completed') - elif _VISUAL_FENCES[kind] in answer: + elif has_visual_block(answer, kind): shown = True else: reasons.append('not_in_answer') diff --git a/application/single_app/functions_orchestration_planner.py b/application/single_app/functions_orchestration_planner.py index 494ce01ee..b71b48d0a 100644 --- a/application/single_app/functions_orchestration_planner.py +++ b/application/single_app/functions_orchestration_planner.py @@ -661,7 +661,12 @@ def resolve_planner_client(settings): capability_availability.visual_outputs.image_proposals is true, image proposal cards the user approves before an AI image is generated. Decide from the request whether a visual materially helps, even unasked, and list it in compose "visuals" (chart, diagram, image_proposal); a chart or -diagram the user asked for is also a chart or diagram deliverable. Plan the gathering each +diagram the user asked for is also a chart or diagram deliverable. A schedule or event timeline +with lanes for activities (such as eating and drinking before, during, and after a ride) is a +diagram, even if the user calls it a chart: declare a diagram deliverable and set compose +"visuals" to ["diagram"]. Preserve the supplied events and relative times; do not invent +measurements or calendar dates to turn a schedule into a numerical plot. Use native charts +for measured values, numerical trends, and category comparisons instead. Plan the gathering each visual needs: exact values for a chart, entities and relationships for a diagram, and concrete visual details for an image. When a chart needs rows an action retrieves, set that action_invoke step's visuals to ["chart"]; it charts the exact rows, and the compose step that diff --git a/docs/explanation/fixes/ORCHESTRATION_TIMELINE_VISUAL_DELIVERY_FIX.md b/docs/explanation/fixes/ORCHESTRATION_TIMELINE_VISUAL_DELIVERY_FIX.md new file mode 100644 index 000000000..fc0bcc7a2 --- /dev/null +++ b/docs/explanation/fixes/ORCHESTRATION_TIMELINE_VISUAL_DELIVERY_FIX.md @@ -0,0 +1,128 @@ +# Orchestration Timeline Visual Delivery Fix + +**Version: 0.261.341** + +Fixed in version: **0.261.341**, recorded in +`application/single_app/config.py`. + +## Issue + +An Orchestrate request asked to turn an earlier gravel-ride nutrition plan into a +two-lane schedule: eating and fueling in one lane, drinking and hydration in the +other, from before the ride through recovery. The answer-writing task completed, +but the plan panel marked the chart **Not delivered** with the reason +"The answer did not include it." + +## Root cause + +The planner could classify an event schedule as a numerical chart because the +user called it a chart. That selected analytical-chart instructions, which +discourage charts for purely textual material and forbid substituting Mermaid +for numerical data charts. A schedule is better represented as a structural +diagram with event labels than as invented numerical series. + +Separately, compose accepted any nonempty Markdown as prepared content. Its +single corrective call covered malformed JSON outputs, not an omitted explicit +visual. Finalization correctly detected the missing chart, but only after the +answer-writing task had finished. A Mermaid schedule declared as a chart also +failed delivery: charts require a `simplechart` fence, whereas diagrams require +a `mermaid` fence. + +The screenshot did not include the complete generated answer, saved plan, or +model response. The investigation reproduced this delivery path locally; it +does not establish precisely why the deployed model omitted the chart. + +## Technical changes + +- `functions_orchestration_planner.py` distinguishes activity schedules and + event timelines from measured numerical plots. Schedule requests use diagram + deliverables and the structured `diagram` visual flag, even when called charts. +- `functions_diagram_operations.py` explains a two-lane representation using + `flowchart TB`, labelled subgraphs, and `direction LR` within each lane. + Relative times remain event labels, without invented calendar dates or values. +- `functions_orchestration_deliverables.py` shares the existing visual-format + presence rule between compose omission checks and final answer delivery. + Compiled deliverable context identifies which explicit, enabled visuals the + compose step owes. +- `functions_orchestration_composition.py` makes at most one application-level + corrective call for either an output-shape problem or a missing explicit chart + or diagram. Correction must still satisfy the declared output kinds and + schemas. Charts already placed from upstream results satisfy the check without + another model call. + +The retry preserves context-budget checks, authorization, cancellation, named +inputs, and hashed logging. Neither generated answer text nor source content is +included in corrective-call logs. No integration action or retrieval is repeated +to correct the prepared answer. + +## Delivery behavior + +Before this fix, a prose-only Markdown reply immediately finished the compose +step and reached finalization with its chart missing. + +After this fix, the step requests a complete corrected answer once. A successful +correction carries the renderable visual into the answer and its delivery state +becomes **Delivered**. Suggested visuals do not cause retries. + +If correction cannot fit the context budget, the original usable answer is kept. +If a schema-valid correction still omits the visual, the corrected usable answer +is kept. Existing finalization reports partial completion and **Not delivered** +for an explicit visual missing from the final answer. Invalid correction output, +provider failures, and revoked access retain their existing explicit failure +paths rather than masquerading as successful correction. + +Saved plans are not silently relabelled. A schedule already declared as a chart +needs replanning to become a diagram. True numerical charts continue to require +their native chart format. + +## Validation + +The regression harness supplies the earlier nutrition plan as conversation +reference data and runs the real schema, executor, result retention, compose, +and final publication services with offline model replies. It covers: + +- omission followed by a delivered two-lane diagram; +- repeated omission retaining a usable answer and partial delivery; +- native-chart correction without accepting Mermaid as a numerical chart; +- JSON-shape and visual corrections sharing one allowance, without a third call; +- malformed corrections, context-budget refusal, and access/cancellation changes + immediately before correction; +- suggested visuals, already-present visuals, and upstream charts avoiding + unnecessary model calls; +- correction logs containing codes and hashed identifiers, not private replies. + +Coverage lives in `test_orchestration_compose_output_retry.py`, +`test_orchestration_visual_outputs.py`, +`test_orchestration_action_chart_delivery.py`, and the existing +`test_mermaid_diagram_prompt_guidance.py` suite. The shared deliverables fixture +also supports retained conversation history for this scenario. + +These four suites passed together: **84 tests passed**. Eight additional +deliverables-contract regressions passed, including shared fixture use, CSV +delivery, visual flag derivation, and answer binding behavior. Documentation +surface coverage passed all seven checks and site quality passed all six checks. +The regression failed before the implementation with the same failed-run/partial- +outcome combination as the reported missing visual. A headless Chromium probe +rendered the two-lane example using the pinned local Mermaid 11.17.2 bundle, +confirmed both lanes and relative-time labels, and made no network requests. + +The local system Python had an incompatible OpenSSL dependency combination; +validation used a separate virtual environment without changing shared packages. +Additional broad deliverables runs encountered native PyMuPDF access violations +in existing Office-renderer tests. Those unrelated renderer paths are not +changed or claimed as validated by this fix. + +## Limitations + +Visual selection and the corrective answer remain model-generated. One correction +is not a guarantee that a visual will be produced, and it must not invent source +data to force success. Format presence is not complete Mermaid syntax validation, +chart semantic validation, or proof that every distinct requested visual appears. +The final answer's existing delivery check remains authoritative. + +The two-lane flowchart shows event sequence and relative-time labels, not exact +time-scaled duration. This fix adds no browser library, chart type, setting, +capability, route, or deployment requirement. + +See [Review and edit orchestration plans](../../guides/review-and-edit-orchestration-plans.md) +for the user-facing behavior. diff --git a/docs/guides/review-and-edit-orchestration-plans.md b/docs/guides/review-and-edit-orchestration-plans.md index dc145ff7d..c40a1db71 100644 --- a/docs/guides/review-and-edit-orchestration-plans.md +++ b/docs/guides/review-and-edit-orchestration-plans.md @@ -4,7 +4,7 @@ title: "Review and edit orchestration plans" description: "Refine proposed work with the planner before running it." section: "Guides" audience: user -version: "0.261.321" +version: "0.261.341" --- ## Decide what should run @@ -141,6 +141,12 @@ a temporary service error. **Visuals** lists the charts, Mermaid diagrams, or image proposal cards the planner asked the answer to include. +Since **0.261.341**, a schedule with activity lanes, such as eating and drinking +before, during, and after a ride, is planned as a **Diagram**, even when you call +it a chart. The diagram preserves the supplied events and relative-time labels; +it shows their order rather than a quantitatively scaled time axis. Numerical +trends and comparisons still use inline data charts. + ### Check what you asked for Since **0.261.138**, **You asked for** appears at the top of the plan panel and on the @@ -157,6 +163,18 @@ The plan panel also names the step that produces each item. **Also included** li the planner added without being asked, such as a suggested chart. Ask the planner for a revision if something you need is missing from the list. +When an answer-writing step omits an explicitly requested chart or diagram, +SimpleChat asks that step to correct its answer once, if the correction fits the +model's context budget. This shares the existing correction allowance for +malformed prepared outputs; it does not rerun source gathering or integration +actions. Suggested visuals do not trigger a correction. + +If a usable answer still omits the requested visual, the answer remains available, +the visual is **Not delivered**, and the run reports partial completion. A +completed answer-writing task alone does not mean its chart or diagram arrived. +For a schedule saved under the wrong visual kind, use a new request or, before +running, **Ask planner** to make it a timeline diagram. + A **Generate image** step shows the image prompt and caption it will use. Each image follows the requested visual style, including photorealistic, and is captioned as an AI-generated image when the plan runs and shows it in the answer. When a report or deck diff --git a/functional_tests/test_orchestration_action_chart_delivery.py b/functional_tests/test_orchestration_action_chart_delivery.py index aeb977522..6c819b74c 100644 --- a/functional_tests/test_orchestration_action_chart_delivery.py +++ b/functional_tests/test_orchestration_action_chart_delivery.py @@ -2,8 +2,9 @@ # test_orchestration_action_chart_delivery.py """ Functional test for orchestration charts that an action draws reaching the user. -Version: 0.261.293 +Version: 0.261.341 Implemented in: 0.261.293 +Carried charts satisfy compose visual correction checks in: 0.261.341 This test ensures that a request such as "Plot BatteryVoltage1 over the last 15 minutes" never ends with a chart that was drawn and then thrown away: @@ -423,6 +424,26 @@ def test_the_answer_step_places_the_chart_from_its_token_without_rereading_its_d assert saved["deliverable_states"] == [{"id": "battery_chart", "state": "delivered"}] +def test_a_carried_chart_satisfies_an_explicit_compose_visual_without_another_call(harness, monkeypatch): + _enable_action(harness, monkeypatch) + plan = _answered_plan() + plan["steps"][0]["delivers"] = [] + plan["steps"][1]["delivers"] = ["battery_chart"] + _store( + harness, _normalize(harness, plan, strict=True), + replies=["BatteryVoltage1 stayed between 27.1 V and 27.4 V."], + ) + + harness.prepare().execute() + + saved = harness.read() + message, _ = _orchestration_metadata(harness) + calls = [call for call in harness.model_calls if call["messages"][0]["content"].startswith("Prepare only")] + assert saved["status"] == "completed" + assert saved["deliverable_states"] == [{"id": "battery_chart", "state": "delivered"}] + assert len(calls) == 1 and message["content"].count("```simplechart") == 1 + + def test_a_chart_that_could_not_be_drawn_is_not_delivered_and_a_retry_draws_it_again(harness, monkeypatch): _enable_action(harness, monkeypatch, with_chart=False) _store( diff --git a/functional_tests/test_orchestration_compose_output_retry.py b/functional_tests/test_orchestration_compose_output_retry.py index 6eae4e753..767bfe53b 100644 --- a/functional_tests/test_orchestration_compose_output_retry.py +++ b/functional_tests/test_orchestration_compose_output_retry.py @@ -1,8 +1,9 @@ # test_orchestration_compose_output_retry.py """Records guidance, JSON mode and the single corrective call for content preparation. -Version: 0.261.141 +Version: 0.261.341 Implemented in: 0.261.141 +Explicit chart/diagram omission correction implemented in: 0.261.341 A request for a CSV of the U.S. states and capitals planned correctly, then failed while preparing content. The compose step was told to write "the finished file", so the model @@ -28,19 +29,44 @@ import pytest from openai import BadRequestError -from test_orchestration_deliverables import _csv_plan, _file_bytes, _files, _plan_request, _saved_steps, _store +from test_orchestration_deliverables import ( + _csv_plan, _file_bytes, _files, _normalize, _plan_request, _saved_steps, _store, +) from test_orchestration_harness_execution import harness, initialized_application # noqa: F401 -from test_support.orchestration_harness_execution import compose_step +from test_support.orchestration_harness_execution import compose_step, input_binding from test_support.versioning import assert_app_version_at_least ROWS = [{"State": "Alabama", "Capital": "Montgomery"}, {"State": "Alaska", "Capital": "Juneau"}] CSV_LINES = ["State,Capital", "Alabama,Montgomery", "Alaska,Juneau"] SECRET_REPLY_TEXT = "PRIVATE_MODEL_REPLY_TEXT" +TIMELINE = """## Eating and drinking schedule + +```mermaid +flowchart TB + subgraph eating["Eating / fueling"] + direction LR + breakfast["Breakfast: 2 hours before"] --> snacks["Snacks: every 30 minutes"] --> meal["Meal: 1 hour after"] + end + subgraph drinking["Drinking / hydration"] + direction LR + before_fluids["Fluids: before the ride"] --> ride_fluids["Fluids: during the ride"] --> after_fluids["Fluids: after the ride"] + end +``` +""" +PRIOR_PLAN = { + "id": "prior-nutrition-plan", "conversation_id": "conversation-1", "role": "assistant", + "content": ( + "For your 40-mile gravel ride, eat breakfast 2 hours before departure, snacks every " + "30 minutes during the ride, and a meal 1 hour after. Drink before, during, and after the ride." + ), + "timestamp": "2026-09-21T17:55:00+00:00", +} def test_version_includes_the_compose_output_retry(): assert_app_version_at_least("0.261.141") + assert_app_version_at_least("0.261.341") def _provider_error(*, param, code, message): @@ -333,3 +359,213 @@ def test_an_empty_reply_is_a_model_failure_and_is_never_retried(harness): assert saved["failure"]["code"] == "model_failed" assert len(_compose_calls(harness)) == 1 and len(harness.replies) == 1 assert _files(harness, ".csv") == [] + + +def _store_visual(harness, replies, *, kind="diagram", requested="explicit", multiple_outputs=False): + step = compose_step("schedule") + step["arguments"].update( + instruction="Transform the supplied prior nutrition plan into eating and drinking lanes with relative times.", + knowledge_basis="general_knowledge", + ) + step["delivers"] = ["answer", "visual"] + if multiple_outputs: + step["outputs"].append({"name": "notes", "kind": "text-v1"}) + plan = _normalize(harness, { + "deliverables": [ + {"id": "answer", "kind": "answer", "requested": "explicit", + "description": "The updated nutrition plan.", "status": "planned"}, + {"id": "visual", "kind": kind, "requested": requested, + "description": "A two-lane eating and drinking timeline.", "status": "planned"}, + ], + "steps": [step], "final_response": input_binding("schedule"), + }) + _store(harness, plan, replies=replies, history=[PRIOR_PLAN]) + return plan + + +def _visual_answers(harness): + return [message for message in harness.assistant_messages() if message["id"] != PRIOR_PLAN["id"]] + + +def test_a_missing_timeline_gets_one_correction_and_reaches_the_answer(harness, monkeypatch): + events = _capture(monkeypatch, "functions_orchestration_composition") + _store_visual(harness, [f"Here is the plan. {SECRET_REPLY_TEXT}", TIMELINE]) + + harness.prepare().execute() + + saved = harness.read() + (answer,) = _visual_answers(harness) + calls = _compose_calls(harness) + assert saved["status"] == saved["outcome"] == "completed" + assert saved["deliverable_states"] == [{"id": "visual", "state": "delivered"}] + assert TIMELINE.strip() in answer["content"] and "Not delivered" not in answer["content"] + assert len(calls) == 2 + assert calls[1]["messages"][:-2] == calls[0]["messages"] + assert calls[1]["messages"][-2]["content"].endswith(SECRET_REPLY_TEXT) + correction = calls[1]["messages"][-1]["content"] + assert "explicitly requested visual" in correction and "```mermaid```" in correction + assert "Return only the complete prepared text." in correction + assert all("response_format" not in call for call in calls) + first_prompt = "\n".join(message["content"] for message in calls[0]["messages"]) + assert PRIOR_PLAN["content"] in first_prompt + assert "[MERMAID_DIAGRAM_GUIDANCE]" in first_prompt + assert "[PROACTIVE_ANALYTICAL_CHART_GUIDANCE]" not in first_prompt + retries = [kwargs["extra"] for _message, kwargs in events + if kwargs.get("extra", {}).get("reason") == "compose_visual_retry"] + assert len(retries) == 1 and retries[0]["step_id_hash"] == _hash("schedule") + assert SECRET_REPLY_TEXT not in repr(events) and "Breakfast" not in repr(events) + + +def test_a_second_visual_omission_keeps_the_answer_and_partial_delivery(harness): + _store_visual(harness, ["First prose answer.", "The revised prose answer."]) + + harness.prepare().execute() + + saved = harness.read() + (answer,) = _visual_answers(harness) + assert saved["outcome"] == "partial" + assert saved["failure"]["code"] == "visual_not_delivered" + assert saved["deliverable_states"] == [{ + "id": "visual", "state": "not_delivered", "message": "The answer did not include it.", + }] + assert "The revised prose answer." in answer["content"] and "Not delivered" in answer["content"] + assert saved["redraw_step_ids"] == ["schedule"] + assert len(_compose_calls(harness)) == 2 + + +def test_a_missing_native_chart_is_corrected_without_accepting_a_diagram(harness): + charts = importlib.import_module("functions_chart_operations") + chart = charts.build_inline_chart_markdown({ + "version": 1, "kind": "line", "chartType": "line", "chartId": "fuel", + "title": "Supplied snack timing", + "data": {"labels": ["Ride"], "datasets": [{"label": "Minutes between snacks", "data": [30]}]}, + }) + _store_visual(harness, [TIMELINE, chart], kind="chart") + + harness.prepare().execute() + + calls = _compose_calls(harness) + assert harness.read()["deliverable_states"] == [{"id": "visual", "state": "delivered"}] + assert len(calls) == 2 and "```simplechart```" in calls[1]["messages"][-1]["content"] + + +@pytest.mark.parametrize(("requested", "reply"), [("suggested", "Only prose."), ("explicit", TIMELINE)]) +def test_suggested_or_already_included_visuals_do_not_trigger_correction(harness, requested, reply): + _store_visual(harness, [reply], requested=requested) + + harness.prepare().execute() + + assert harness.read()["status"] == "completed" + assert len(_compose_calls(harness)) == 1 + + +@pytest.mark.parametrize("first_reply", [ + "Not a JSON object.", + json.dumps({"answer": "Only prose.", "notes": "Keep existing quantities."}), +]) +def test_shape_and_visual_corrections_share_one_retry(harness, first_reply): + final_reply = json.dumps({"answer": TIMELINE, "notes": "Keep existing quantities."}) + _store_visual(harness, [first_reply, final_reply], multiple_outputs=True) + + harness.prepare().execute() + + calls = _compose_calls(harness) + assert harness.read()["deliverable_states"] == [{"id": "visual", "state": "delivered"}] + assert len(calls) == 2 + assert all(call["response_format"] == {"type": "json_object"} for call in calls) + assert "```mermaid```" in calls[1]["messages"][-1]["content"] + correction = calls[1]["messages"][-1]["content"] + assert "JSON object" in correction and '"answer"' in correction and '"notes"' in correction + + +def test_a_shape_correction_that_still_omits_the_visual_does_not_get_a_third_call(harness): + _store_visual(harness, ["Not JSON.", json.dumps({"answer": "Still prose.", "notes": "Notes."})], + multiple_outputs=True) + + harness.prepare().execute() + + (answer,) = _visual_answers(harness) + assert harness.read()["outcome"] == "partial" and "Still prose." in answer["content"] + assert len(_compose_calls(harness)) == 2 + + +def test_a_visual_correction_must_still_match_the_declared_json_outputs(harness): + _store_visual(harness, [json.dumps({"answer": "Only prose.", "notes": "Notes."}), "Not JSON."], + multiple_outputs=True) + + harness.prepare().execute() + + assert _saved_steps(harness)["schedule"]["failure"]["code"] == "result_invalid" + assert len(_compose_calls(harness)) == 2 + + +def test_a_visual_correction_that_would_not_fit_keeps_the_valid_partial_answer(harness, monkeypatch): + _store_visual(harness, ["The usable prose answer."]) + budget = importlib.import_module("functions_workflow_context") + original = budget.calculate_workflow_context_budget + audits = [] + + def audit(messages, *args, **kwargs): + result = original(messages, *args, **kwargs) + audits.append(len(messages)) + return {**result, "decision": "full_input" if len(audits) == 1 else "reduced_input"} + + monkeypatch.setattr(budget, "calculate_workflow_context_budget", audit) + + harness.prepare().execute() + + (answer,) = _visual_answers(harness) + assert harness.read()["outcome"] == "partial" and "The usable prose answer." in answer["content"] + assert len(_compose_calls(harness)) == 1 + assert len(audits) == 2 and audits[1] == audits[0] + 2 + + +@pytest.mark.parametrize("stop", ["access", "cancellation"]) +def test_visual_correction_rechecks_access_and_cancellation_before_another_call(harness, monkeypatch, stop): + _store_visual(harness, ["Only prose.", TIMELINE]) + execution = harness.prepare() + budget = importlib.import_module("functions_workflow_context") + original = budget.calculate_workflow_context_budget + audits = [] + cancelled = [] + + def audit(messages, *args, **kwargs): + result = original(messages, *args, **kwargs) + audits.append(len(messages)) + if len(audits) == 2: + if stop == "access": + harness.conversations.items[("conversation-1", "conversation-1")]["user_id"] = "another-owner" + else: + cancelled.append(True) + return result + + monkeypatch.setattr(budget, "calculate_workflow_context_budget", audit) + monkeypatch.setattr(execution.lease, "cancel_requested", lambda: bool(cancelled)) + + execution.execute() + + assert len(audits) == 2 + assert len(_compose_calls(harness)) == 1 and harness.replies == [TIMELINE] + answers = _visual_answers(harness) + assert not any("```mermaid" in message["content"] for message in answers) + + +def test_visual_omission_checks_use_only_explicit_enabled_markdown_deliverables(initialized_application): + deliverables = importlib.import_module("functions_orchestration_deliverables") + step = { + "outputs": [{"name": "answer", "kind": "markdown-v1"}, {"name": "notes", "kind": "text-v1"}], + "deliverable_context": [ + {"relation": "delivers", "requested": "explicit", "kind": "diagram"}, + {"relation": "delivers", "requested": "suggested", "kind": "chart"}, + {"relation": "unavailable", "requested": "explicit", "kind": "chart"}, + {"relation": "delivers", "requested": "explicit", "kind": "image"}, + {"relation": "delivers", "requested": "explicit", "kind": "chart", "enabled": False}, + ], + } + missing = deliverables.missing_compose_visuals(step, {"answer": "Prose.", "notes": TIMELINE}) + assert missing == ["diagram"] + missing = deliverables.missing_compose_visuals(step, {"answer": TIMELINE}) + assert missing == [] + step["enabled"] = False + missing = deliverables.missing_compose_visuals(step, {"answer": "Prose."}) + assert missing == [] diff --git a/functional_tests/test_orchestration_deliverables.py b/functional_tests/test_orchestration_deliverables.py index fdb378de2..e44734e97 100644 --- a/functional_tests/test_orchestration_deliverables.py +++ b/functional_tests/test_orchestration_deliverables.py @@ -1,11 +1,12 @@ # test_orchestration_deliverables.py """The deliverables contract and planned image generation in Gather / Reason / Render plans. -Version: 0.261.192 +Version: 0.261.341 Implemented in: 0.261.138 Single orchestration contract updated in: 0.261.139 Planner kind-specific fields and absent answer bindings updated in: 0.261.140 Row-shaped compose guidance and JSON mode for CSV scenarios updated in: 0.261.141 +Prior-answer visual correction fixture support added in: 0.261.341 Uses the initialized headless harness (real bootstrap, planner, schema, executor, result store, chat image persistence, rendering service and Office renderers) with the planning @@ -176,20 +177,22 @@ def _president_plan(*, with_file=True, with_search=False, output_format="docx"): return {"deliverables": deliverables, "steps": steps, "final_response": input_binding("report", "report")} -def _store(harness, plan, *, replies=()): +def _store(harness, plan, *, replies=(), history=()): """Save a validated plan as run-1, exactly as the harness saves its own fixtures.""" run_store = importlib.import_module("functions_orchestration_runs") context = importlib.import_module("functions_orchestration_context") memory = importlib.import_module("functions_orchestration_memory") plan = deepcopy(plan) plan.update(run_id="run-1", plan_id="plan-1", turn_id="turn-1") + for message in history: + harness.messages.create_item(message) normalized = context.normalize_history_message(harness.turn) turn_context = { "turn_id": "turn-1", "user_message": harness.turn["content"], "user_message_id": harness.turn["id"], "user_message_fingerprint": normalized["fingerprint"], "resolved_message": harness.turn["content"], "seeds": {}, "original_seeds": {}, "answered_questions": [], "planning_token_usage": {}, - "conversation_context": context.build_conversation_snapshot([], harness.settings), + "conversation_context": context.build_conversation_snapshot(history, harness.settings), "memory_audience": memory.validate_memory_audience(harness.conversation, "owner"), "memory_scope": None, } diff --git a/functional_tests/test_orchestration_visual_outputs.py b/functional_tests/test_orchestration_visual_outputs.py index f030e705e..2950214c9 100644 --- a/functional_tests/test_orchestration_visual_outputs.py +++ b/functional_tests/test_orchestration_visual_outputs.py @@ -2,10 +2,11 @@ # test_orchestration_visual_outputs.py """ Functional test for charts, Mermaid diagrams, and image proposals in orchestrated answers. -Version: 0.261.270 +Version: 0.261.341 Implemented in: 0.261.132 Single orchestration contract updated in: 0.261.139 Action steps run without an invocation capture in: 0.261.270 +Schedule diagrams distinguished from numerical charts in: 0.261.341 This test ensures orchestration carries chart payloads safely, builds visual guidance only from planner-declared visual flags, preserves saved instruction memory precedence, @@ -195,6 +196,14 @@ def test_gathering_and_agent_notes_keep_what_each_visual_needs(): assert visuals.agent_visual_note({}) == '' +def test_schedule_diagram_guidance_preserves_relative_times_without_chart_instructions(): + guidance = '\n'.join(visuals.build_answer_visual_guidance({'diagram': True})) + assert 'two-lane' in guidance and 'relative times' in guidance + assert 'flowchart TB' in guidance and 'direction LR' in guidance + assert 'invent calendar dates' in guidance + assert charts.PROACTIVE_CHART_GUIDANCE_MARKER not in guidance + + def test_only_instruction_memories_reach_visual_preferences(): instruction = {'role': 'system', 'content': '\n- Never make charts.\n'} fact = {'role': 'system', 'content': '\n- Private fact.\n'} @@ -341,6 +350,9 @@ def test_planner_prompt_describes_visual_outputs_for_the_single_contract(modules assert 'set that action_invoke step\'s visuals to ["chart"]' in prompt assert 'Web search returns text and links only' in prompt assert 'Saved instructions in memory about visuals' in prompt + assert 'schedule or event timeline' in prompt + assert 'even if the user calls it a chart' in prompt + assert 'diagram deliverable' in prompt compose_capability = modules.registry.get_capability('compose') assert 'visuals' in json.dumps(compose_capability)