diff --git a/mellea/backends/adapters/catalog.py b/mellea/backends/adapters/catalog.py index eb07c19a0..54e33f3a0 100644 --- a/mellea/backends/adapters/catalog.py +++ b/mellea/backends/adapters/catalog.py @@ -64,6 +64,7 @@ class IntriniscsCatalogEntry(pydantic.BaseModel): ############################################ # Core Intrinsics ############################################ + IntriniscsCatalogEntry(name="context-attribution", repo_id=_CORE_R1_REPO), IntriniscsCatalogEntry(name="requirement-check", repo_id=_CORE_R1_REPO), IntriniscsCatalogEntry( name="requirement_check", repo_id=_CORE_REPO diff --git a/mellea/formatters/granite/intrinsics/input.py b/mellea/formatters/granite/intrinsics/input.py index c578c62d2..bfa064ae2 100644 --- a/mellea/formatters/granite/intrinsics/input.py +++ b/mellea/formatters/granite/intrinsics/input.py @@ -48,23 +48,23 @@ def sentence_delimiter(tag, sentence_num) -> str: def mark_sentence_boundaries( - split_strings: list[list[str]], tag_prefix: str -) -> list[str]: + split_strings: list[list[str]], tag_prefix: str, index: int = 0 +) -> tuple[list[str], int]: """Modify input strings by inserting sentence boundary markers. Modify one or more input strings by inserting a tag in the form ``<[prefix][number]>`` at the location of each sentence boundary. - Args: - split_strings: Input string(s), pre-split into sentences. - tag_prefix: String to place before the number part of each tagged - sentence boundary. + :param split_strings: Input string(s), pre-split into sentences + :param tag_prefix: String to place before the number part of each tagged + sentence boundary. + :param index: Starting index for sentence numbering. Defaults to 0. Pass a + non-zero value to continue numbering from a prior call. - Returns: - List of input strings with all sentence boundaries marked. + :returns: Tuple of (list of input strings with all sentence boundaries marked, + next available index after the last sentence). """ - index = 0 result: list[str] = [] for sentences in split_strings: to_concat = [] @@ -72,7 +72,7 @@ def mark_sentence_boundaries( to_concat.append(f"{sentence_delimiter(tag_prefix, index)}{sentence}") index += 1 result.append(" ".join(to_concat)) - return result + return result, index def move_documents_to_message( @@ -291,10 +291,11 @@ def __init__( f"Received {self.sentence_boundaries}." ) for k, v in self.sentence_boundaries.items(): - if k not in ("last_message", "documents"): + if k not in ("last_message", "documents", "all_but_last_message"): raise ValueError( f"Unexpected location '{k}' in 'sentence_boundaries' field. " - f"Value should be 'last_message' or 'documents'." + f"Value should be 'last_message', 'documents', or " + f"'all_but_last_message'." ) if not isinstance(v, str): raise TypeError( @@ -324,6 +325,8 @@ def _mark_sentence_boundaries( :rtype: ChatCompletion """ # Mark sentence boundaries in the last message. + # last_message uses its own numbering starting from 0, independent of + # the numbering used for documents and conversation history. if self.sentence_boundaries and "last_message" in self.sentence_boundaries: messages = chat_completion.messages.copy() # Do not modify input! last_message_as_sentences = list( @@ -331,14 +334,17 @@ def _mark_sentence_boundaries( ) last_message_tag = self.sentence_boundaries["last_message"] if last_message_tag: - rewritten_last_message_text = mark_sentence_boundaries( + rewritten_texts, _ = mark_sentence_boundaries( [last_message_as_sentences], last_message_tag - )[0] - messages[-1].content = rewritten_last_message_text + ) + messages[-1].content = rewritten_texts[0] chat_completion = chat_completion.model_copy( update={"messages": messages} ) + # documents and all_but_last_message share a continuous numbering. + index = 0 + # Mark sentence boundaries in documents if present if ( chat_completion.extra_body @@ -355,11 +361,14 @@ def _mark_sentence_boundaries( # where `k` is the number of sentences in ALL documents. documents_tag = self.sentence_boundaries["documents"] if documents_tag: + rewritten_texts, index = mark_sentence_boundaries( + docs_as_sentences, documents_tag, index + ) rewritten_docs = [ doc.model_copy(update={"text": text}) for doc, text in zip( chat_completion.extra_body.documents, - mark_sentence_boundaries(docs_as_sentences, documents_tag), + rewritten_texts, strict=True, ) ] @@ -370,6 +379,30 @@ def _mark_sentence_boundaries( chat_completion = chat_completion.model_copy( update={"extra_body": extra_body} ) + + # Mark sentence boundaries in conversation history if requested. + # Uses the same numbering as documents, continuing from where they left off. + if ( + self.sentence_boundaries + and "all_but_last_message" in self.sentence_boundaries + ): + history_tag = self.sentence_boundaries["all_but_last_message"] + if history_tag: + messages = chat_completion.messages.copy() # Do not modify input! + for i, message in enumerate(messages[:-1]): + msg_as_sentences = list( + self.sentence_splitter.tokenize(message.content) + ) + rewritten_texts, index = mark_sentence_boundaries( + [msg_as_sentences], history_tag, index + ) + messages[i] = message.model_copy( + update={"content": rewritten_texts[0]} + ) + chat_completion = chat_completion.model_copy( + update={"messages": messages} + ) + return chat_completion def _transform( diff --git a/mellea/formatters/granite/intrinsics/output.py b/mellea/formatters/granite/intrinsics/output.py index aaa3efdae..a16f8a31b 100644 --- a/mellea/formatters/granite/intrinsics/output.py +++ b/mellea/formatters/granite/intrinsics/output.py @@ -515,25 +515,32 @@ def __init__( config: dict, input_path_expr: list[str | int | None], /, - source: str, + source: str | list[str], output_names: dict, ): """Initialize DecodeSentences with a source location and output field name mapping. + :param source: Name (or list of names) of the location(s) to look for + sentences; each name can be "last_message", "documents", or + "all_but_last_message". + :param output_names: Names of new result fields to add + Raises: - ValueError: If ``source`` is not ``"last_message"`` or - ``"documents"``, or if an unexpected key is found in - ``output_names``. + ValueError: If ``source`` is not one of the allowed values, or if + an unexpected key is found in ``output_names``. TypeError: If ``output_names`` is not a dict. """ super().__init__(config, input_path_expr) - allowed_sources = ("last_message", "documents") - if source not in allowed_sources: - raise ValueError( - f"'source' argument must be one of {allowed_sources}. " - f"Received '{source}'" - ) + if isinstance(source, str): + source = [source] + allowed_sources = ("last_message", "documents", "all_but_last_message") + for s in source: + if s not in allowed_sources: + raise ValueError( + f"'source' argument must be one of {allowed_sources}. " + f"Received '{s}'" + ) self.source = source if not isinstance(output_names, dict): @@ -541,7 +548,9 @@ def __init__( f"Expected mapping for output_names, but received {output_names}" ) for k in output_names: - if source == "documents" and k == "document_id": + if "documents" in source and k == "document_id": + continue + if "all_but_last_message" in source and k == "message_index": continue if k not in ("begin", "end", "text"): raise ValueError(f"Unexpected key '{k}' in output_names") @@ -551,6 +560,7 @@ def __init__( self.end_name = output_names.get("end") self.text_name = output_names.get("text") self.document_id_name = output_names.get("document_id") + self.message_index_name = output_names.get("message_index") if config["docs_as_message"] and config["docs_as_message"] not in [ "json", @@ -575,77 +585,114 @@ def _prepare( f"'{self.rule_name()}' rule requires this object." ) - if self.source == "documents": - tag = self.config["sentence_boundaries"]["documents"] - if tag is None: - raise ValueError( - f"'{self.rule_name()}' attempting to decode document sentences, " - f"but 'sentence_boundaries' section of config file is missing " - f"the entry that tells how to tag document sentence boundaries." - ) + begins: list[int] = [] + ends: list[int] = [] + texts: list[str] = [] + document_ids: list[str | None] = [] + message_indices: list[int | None] = [] + next_sentence_num = 0 + + for src in self.source: + if src == "documents": + tag = self.config["sentence_boundaries"]["documents"] + if tag is None: + raise ValueError( + f"'{self.rule_name()}' attempting to decode document sentences, " + f"but 'sentence_boundaries' section of config file is missing " + f"the entry that tells how to tag document sentence boundaries." + ) - documents: list[Document] = [] - if not self.config["docs_as_message"]: - # Most common path: Documents from extra_body - if chat_completion.extra_body is not None: - documents = chat_completion.extra_body.documents or [] - else: - # Model requires documents in a user message. Decode the message. - if self.config["docs_as_message"] == "json": - documents_json = json.loads(chat_completion.messages[0].content) - documents = [Document.model_validate(d) for d in documents_json] - elif self.config["docs_as_message"] == "roles": - for message in chat_completion.messages: - if message.role.startswith("document "): - document = Document( - doc_id=message.role[len("document ") :], - text=message.content, - ) - documents.append(document) + documents: list[Document] = [] + if not self.config["docs_as_message"]: + # Most common path: Documents from extra_body + if chat_completion.extra_body is not None: + documents = chat_completion.extra_body.documents or [] else: + # Model requires documents in a user message. Decode the message. + if self.config["docs_as_message"] == "json": + documents_json = json.loads(chat_completion.messages[0].content) + documents = [Document.model_validate(d) for d in documents_json] + elif self.config["docs_as_message"] == "roles": + for message in chat_completion.messages: + if message.role.startswith("document "): + document = Document( + doc_id=message.role[len("document ") :], + text=message.content, + ) + documents.append(document) + else: + raise ValueError( + f"Unsupported doc type {self.config['docs_as_message']}" + ) + + # De-split sentences; numbers start at next_sentence_num and continue + # across documents. + for d in documents: + local_results = _desplit_sentences(d.text, tag, next_sentence_num) + num_local_sentences = len(local_results["begins"]) + begins.extend(local_results["begins"]) + ends.extend(local_results["ends"]) + texts.extend(local_results["texts"]) + document_ids.extend([d.doc_id] * num_local_sentences) + message_indices.extend([None] * num_local_sentences) + next_sentence_num += num_local_sentences + + elif src == "last_message": + tag = self.config["sentence_boundaries"]["last_message"] + if tag is None: raise ValueError( - f"Unsupported doc type {self.config['docs_as_message']}" + f"'{self.rule_name()}' attempting to decode the last message, " + f"but 'sentence_boundaries' section of config file is missing " + f"the entry that tells how to tag message sentence boundaries." ) - # De-split the sentences in each document in turn. Sentence numbers - # start at zero on the first document and continue in subsequent documents. - begins = [] - ends = [] - texts = [] - document_ids = [] - - next_sentence_num = 0 - for d in documents: - local_results = _desplit_sentences(d.text, tag, next_sentence_num) + # Use second-to-last turn if the input processing added an instruction turn + message_ix = -2 if self.config["instruction"] else -1 + target_text = chat_completion.messages[message_ix].content + local_results = _desplit_sentences(target_text, tag, next_sentence_num) num_local_sentences = len(local_results["begins"]) begins.extend(local_results["begins"]) ends.extend(local_results["ends"]) texts.extend(local_results["texts"]) - document_ids.extend([d.doc_id] * num_local_sentences) + document_ids.extend([None] * num_local_sentences) + message_indices.extend([None] * num_local_sentences) next_sentence_num += num_local_sentences - return { - "begins": begins, - "ends": ends, - "texts": texts, - "document_ids": document_ids, - } - if self.source == "last_message": - tag = self.config["sentence_boundaries"]["last_message"] - if tag is None: - raise ValueError( - f"'{self.rule_name()}' attempting to decode the last message, " - f"but 'sentence_boundaries' section of config file is missing " - f"the entry that tells how to tag message sentence boundaries." - ) + elif src == "all_but_last_message": + tag = self.config["sentence_boundaries"]["all_but_last_message"] + if tag is None: + raise ValueError( + f"'{self.rule_name()}' attempting to decode conversation " + f"history sentences, but 'sentence_boundaries' section of " + f"config file is missing the entry that tells how to tag " + f"all_but_last_message sentence boundaries." + ) - # Use second-to-last turn if the input processing added an instruction turn - message_ix = -2 if self.config["instruction"] else -1 - target_text = chat_completion.messages[message_ix].content + # Use second-to-last as the boundary if an instruction turn was added + last_ix = -2 if self.config["instruction"] else -1 + history_messages = chat_completion.messages[:last_ix] + for i, message in enumerate(history_messages): + local_results = _desplit_sentences( + message.content, tag, next_sentence_num + ) + num_local_sentences = len(local_results["begins"]) + begins.extend(local_results["begins"]) + ends.extend(local_results["ends"]) + texts.extend(local_results["texts"]) + document_ids.extend([None] * num_local_sentences) + message_indices.extend([i] * num_local_sentences) + next_sentence_num += num_local_sentences - return _desplit_sentences(target_text, tag, 0) + else: + raise ValueError(f"Unexpected source string '{src}'") - raise ValueError(f"Unexpected source string '{self.source}'") + return { + "begins": begins, + "ends": ends, + "texts": texts, + "document_ids": document_ids, + "message_indices": message_indices, + } def _transform(self, value: Any, path: tuple, prepare_output: dict) -> dict: # Unpack global values we set aside during the prepare phase @@ -653,6 +700,7 @@ def _transform(self, value: Any, path: tuple, prepare_output: dict) -> dict: ends = prepare_output["ends"] texts = prepare_output["texts"] document_ids = prepare_output.get("document_ids") + message_indices = prepare_output.get("message_indices") if not isinstance(value, int): raise TypeError( @@ -670,6 +718,8 @@ def _transform(self, value: Any, path: tuple, prepare_output: dict) -> dict: result[self.text_name] = texts[sentence_num] if self.document_id_name is not None: result[self.document_id_name] = document_ids[sentence_num] # type: ignore[index] + if self.message_index_name is not None: + result[self.message_index_name] = message_indices[sentence_num] # type: ignore[index] return result diff --git a/test/formatters/granite/test_intrinsics_formatters.py b/test/formatters/granite/test_intrinsics_formatters.py index 29de4270b..66bc8c5bc 100644 --- a/test/formatters/granite/test_intrinsics_formatters.py +++ b/test/formatters/granite/test_intrinsics_formatters.py @@ -205,6 +205,12 @@ def _maybe_download_yaml(self): # task="citations", # is_alora=True, # ), + YamlJsonCombo( + short_name="context-attribution", + inputs_file=_INPUT_JSON_DIR / "context-attribution.json", + task="context-attribution", + repo_id="ibm-granite/granitelib-core-r1.0", + ), # gpt-oss-20b intrinsics (canned output tests only, no inference) YamlJsonCombo( short_name="gpt_oss_answerability", @@ -261,12 +267,17 @@ def _maybe_download_yaml(self): } # Combinations suitable for an Ollama backend +_NO_OLLAMA_ADAPTER = { + # Ollama LoRA adapter not yet available on HF Hub + "context-attribution" +} _YAML_JSON_COMBOS_FOR_OLLAMA = { k: v for k, v in _YAML_JSON_COMBOS.items() if v.task is not None and not v.is_alora and v.base_model_id == "ibm-granite/granite-4.0-micro" + and k not in _NO_OLLAMA_ADAPTER } diff --git a/test/formatters/granite/testdata/input_json/context-attribution.json b/test/formatters/granite/testdata/input_json/context-attribution.json new file mode 100644 index 000000000..1da0ad738 --- /dev/null +++ b/test/formatters/granite/testdata/input_json/context-attribution.json @@ -0,0 +1,37 @@ +{ + "messages": [ + { + "role": "user", + "content": "Who were the members of The Metal Ono Band, which was formed by Yoko Ono in 1976 to explore her interest in heavy metal music?" + }, + { + "role": "assistant", + "content": "I'm sorry, but I don't have the data to answer that specific question. " + }, + { + "role": "user", + "content": "What was the concept behind the formation of the Plastic Ono Band?" + }, + { + "role": "assistant", + "content": "The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969." + } + ], + "temperature": 0.0, + "extra_body": { + "documents": [ + { + "doc_id": "0", + "text": "The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. Lennon and Ono had begun a personal and artistic relationship in 1968, collaborating on several experimental releases. Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. The band would go on to feature a rotating lineup of many musicians, including Eric Clapton, Klaus Voormann, Alan White, Billy Preston, Jim Keltner, Delaney & Bonnie and Friends, and Lennon's former Beatles bandmates George Harrison and Ringo Starr. Lennon and Ono left the UK to settle in New York City during the fall of 1971. In Greenwich Village, the couple became more politically active and began writing protest songs. These songs became the basis for their next album, Some Time in New York City. As backing, they enlisted the help of New York band Elephant's Memory, consisting of guitarist Wayne 'Tex' Gabriel, bassist Gary Van Scyoc, saxophonist Stan Bronstein, keyboardist Adam Ippolito, keyboardist John La Boosca, and drummer Richard Frank, Jr. Phil Spector produced, and Jim Keltner also played on the album. The album was released on 12 June 1972, credited to \"John & Yoko/Plastic Ono Band with Elephant's Memory plus Invisible Strings\". Some Time in New York City included a second disc, entitled Live Jam, which included the recordings from the 1969 Peace for Christmas concert and the 1971 performance with Frank Zappa. Ono and Lennon continued their work with Elephant's Memory throughout 1972, performing as the Plastic Ono Elephant's Memory Band (which also included Jim Keltner). On 30 August, they performed a pair of benefit concerts at Madison Square Garden. The benefit, entitled \"One to One\", was organised by Geraldo Rivera to raise money for children with mental challenges. By this time, La Boosca had departed the band, and the concert saw the addition of John Ward on bass. The concert was filmed and recorded, later released in February 1986 as the album Live In New York City. They also performed at the Jerry Lewis MDA Labor Day Telethon. The last collaboration of the Plastic Ono Elephant's Memory Band was Ono's double album Approximately Infinite Universe. It was recorded throughout the fall of 1972, and was released in January 1973." + }, + { + "doc_id": "1", + "text": "The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. Lennon and Ono had begun a personal and artistic relationship in 1968, collaborating on several experimental releases. Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. The band would go on to feature a rotating lineup of many musicians, including Eric Clapton, Klaus Voormann, Alan White, Billy Preston, Jim Keltner, Delaney & Bonnie and Friends, and Lennon's former Beatles bandmates George Harrison and Ringo Starr. By the beginning of 1973, recording had begun in earnest on Ono's next album, Feeling the Space, featuring a new group of studio musicians. The newest incarnation of the Plastic Ono Band featured guitarist David Spinozza, keyboardist Ken Ascher, bassist Gordon Edwards, percussionists Arthur Jenkins and David Friedman, saxophonist Michael Brecker, pedal steel guitarist Sneaky Pete Kleinow, as well as regular contributor Jim Keltner. The album would be released in November. Throughout 1973, Lennon and Ono's relationship became strained. By August, the two had begun a period of separation that Lennon called \"The Lost Weekend\". Lennon began the recording of his own album, Mind Games, using the same players as on Feeling the Space, dubbed \"The Plastic U.F.Ono Band\". Around the time of the album's release in November, Lennon moved to Los Angeles with new lover May Pang. In October, Lennon began the recording of an album of rock 'n' roll oldies (a contractual obligation due to a lawsuit). These featured many Plastic Ono Band regulars (including much of the \"U.F.Ono Band\", Klaus Voorman, and the return of Phil Spector to the production chair), but upon release in 1975 as Rock 'n' Roll, it was credited to Lennon alone. The sessions for Rock 'n' Roll were extremely troubled, and the sessions were abandoned until a later date. In July 1974, Lennon returned to New York to record Walls and Bridges. The new \"Plastic Ono Nuclear Band\" featured both old and new faces, with Jim Keltner, Kenneth Ascher, and Arthur Jenkins continuing from Mind Games, the returns of Klaus Voorman, Nicky Hopkins, and Bobby Keys, and the addition of guitarists Jesse Ed Davis and Eddie Mottau. Recording was finished in August, and the album was released 26 September and 4 October in the US and UK respectively. Walls and Bridges would prove to be the last release of new material by the Plastic Ono Band in the 1970s. Lennon subsequently returned to his marriage with Ono and retired from music following the birth of his son Sean. The compilation Shaved Fish was released in October 1975, Lennon's last release credited to the Plastic Ono Band. Upon his and Ono's return to music in 1980 for the album Double Fantasy, they played with an all-new group of studio musicians who were not billed as any variation of the Plastic Ono Band name. Lennon was shot and killed shortly after the release of the album." + }, + { + "doc_id": "2", + "text": "John Winston Ono Lennon (9 October 1940 - 8 December 1980) was an English singer, songwriter, and peace activist who co-founded the Beatles, the most commercially successful band in the history of popular music. He and fellow member Paul McCartney formed a much-celebrated songwriting partnership. Along with George Harrison and Ringo Starr, the group would ascend to world-wide fame during the 1960s. During his marriage to Cynthia, Lennon's first son Julian was born at the same time that his commitments with the Beatles were intensifying at the height of Beatlemania. Lennon was touring with the Beatles when Julian was born on 8 April 1963. Julian's birth, like his mother Cynthia's marriage to Lennon, was kept secret because Epstein was convinced that public knowledge of such things would threaten the Beatles' commercial success. Julian recalled that as a small child in Weybridge some four years later, \"I was trundled home from school and came walking up with one of my watercolour paintings. It was just a bunch of stars and this blonde girl I knew at school. And Dad said, 'What's this?' I said, 'It's Lucy in the sky with diamonds.'\" Lennon used it as the title of a Beatles song, and though it was later reported to have been derived from the initials LSD, Lennon insisted, \"It's not an acid song.\" McCartney corroborated Lennon's explanation that Julian innocently came up with the name. Lennon was distant from Julian, who felt closer to McCartney than to his father. During a car journey to visit Cynthia and Julian during Lennon's divorce, McCartney composed a song, \"Hey Jules\", to comfort him. It would evolve into the Beatles song \"Hey Jude\". Lennon later said, \"That's his best song. It started off as a song about my son Julian ... he turned it into 'Hey Jude'. I always thought it was about me and Yoko but he said it wasn't.\" Lennon's relationship with Julian was already strained, and after Lennon and Ono moved to Manhattan in 1971, Julian would not see his father again until 1973. With Pang's encouragement, arrangements were made for Julian (and his mother) to visit Lennon in Los Angeles, where they went to Disneyland. Julian started to see his father regularly, and Lennon gave him a drumming part on a Walls and Bridges track. He bought Julian a Gibson Les Paul guitar and other instruments, and encouraged his interest in music by demonstrating guitar chord techniques. Julian recalls that he and his father \"got on a great deal better\" during the time he spent in New York: \"We had a lot of fun, laughed a lot and had a great time in general.\" In a Playboy interview with David Sheff shortly before his death, Lennon said, \"Sean was a planned child, and therein lies the difference. I don't love Julian any less as a child. He's still my son, whether he came from a bottle of whiskey or because they didn't have pills in those days. He's here, he belongs to me, and he always will.\" He said he was trying to re-establish a connection with the then 17-year-old, and confidently predicted, \"Julian and I will have a relationship in the future.\" After his death it was revealed that he had left Julian very little in his will." + } + ] + } +} \ No newline at end of file diff --git a/test/formatters/granite/testdata/test_canned_input/context-attribution.json b/test/formatters/granite/testdata/test_canned_input/context-attribution.json new file mode 100644 index 000000000..977ca9d4f --- /dev/null +++ b/test/formatters/granite/testdata/test_canned_input/context-attribution.json @@ -0,0 +1,76 @@ +{ + "messages": [ + { + "content": " Who were the members of The Metal Ono Band, which was formed by Yoko Ono in 1976 to explore her interest in heavy metal music?", + "role": "user" + }, + { + "content": " I'm sorry, but I don't have the data to answer that specific question.", + "role": "assistant" + }, + { + "content": " What was the concept behind the formation of the Plastic Ono Band?", + "role": "user" + }, + { + "content": " The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.", + "role": "assistant" + }, + { + "content": "You provided the last assistant response above based on context, which may include documents and/or previous conversation turns. Your response is divided into sentences, numbered in the format sentence 0 sentence 1 ... Sentences in the context are also numbered: sentence 0 sentence 1 ... For each response sentence, please list the context sentences that were most important for you to generate the response sentence. Provide your answer in JSON format, as an array of JSON objects, where each object has two members: \"r\" with the response sentence number as the value, and \"c\" with an array of context sentence numbers as the value. An example of such an array of objects is [{\"r\": 0, \"c\": [3, 1, 4]}, {\"r\": 1, \"c\": [1, 5]}]. List the context sentences in order from most important to least important. Ensure that you include an object for each response sentence, even if the corresponding array of context sentence numbers is empty. Answer with only the JSON and do not explain.\n", + "role": "user" + } + ], + "extra_body": { + "documents": [ + { + "text": " The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. Lennon and Ono had begun a personal and artistic relationship in 1968, collaborating on several experimental releases. Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. The band would go on to feature a rotating lineup of many musicians, including Eric Clapton, Klaus Voormann, Alan White, Billy Preston, Jim Keltner, Delaney & Bonnie and Friends, and Lennon's former Beatles bandmates George Harrison and Ringo Starr. Lennon and Ono left the UK to settle in New York City during the fall of 1971. In Greenwich Village, the couple became more politically active and began writing protest songs. These songs became the basis for their next album, Some Time in New York City. As backing, they enlisted the help of New York band Elephant's Memory, consisting of guitarist Wayne 'Tex' Gabriel, bassist Gary Van Scyoc, saxophonist Stan Bronstein, keyboardist Adam Ippolito, keyboardist John La Boosca, and drummer Richard Frank, Jr. Phil Spector produced, and Jim Keltner also played on the album. The album was released on 12 June 1972, credited to \"John & Yoko/Plastic Ono Band with Elephant's Memory plus Invisible Strings\". Some Time in New York City included a second disc, entitled Live Jam, which included the recordings from the 1969 Peace for Christmas concert and the 1971 performance with Frank Zappa. Ono and Lennon continued their work with Elephant's Memory throughout 1972, performing as the Plastic Ono Elephant's Memory Band (which also included Jim Keltner). On 30 August, they performed a pair of benefit concerts at Madison Square Garden. The benefit, entitled \"One to One\", was organised by Geraldo Rivera to raise money for children with mental challenges. By this time, La Boosca had departed the band, and the concert saw the addition of John Ward on bass. The concert was filmed and recorded, later released in February 1986 as the album Live In New York City. They also performed at the Jerry Lewis MDA Labor Day Telethon. The last collaboration of the Plastic Ono Elephant's Memory Band was Ono's double album Approximately Infinite Universe. It was recorded throughout the fall of 1972, and was released in January 1973.", + "doc_id": "0" + }, + { + "text": " The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. Lennon and Ono had begun a personal and artistic relationship in 1968, collaborating on several experimental releases. Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. The band would go on to feature a rotating lineup of many musicians, including Eric Clapton, Klaus Voormann, Alan White, Billy Preston, Jim Keltner, Delaney & Bonnie and Friends, and Lennon's former Beatles bandmates George Harrison and Ringo Starr. By the beginning of 1973, recording had begun in earnest on Ono's next album, Feeling the Space, featuring a new group of studio musicians. The newest incarnation of the Plastic Ono Band featured guitarist David Spinozza, keyboardist Ken Ascher, bassist Gordon Edwards, percussionists Arthur Jenkins and David Friedman, saxophonist Michael Brecker, pedal steel guitarist Sneaky Pete Kleinow, as well as regular contributor Jim Keltner. The album would be released in November. Throughout 1973, Lennon and Ono's relationship became strained. By August, the two had begun a period of separation that Lennon called \"The Lost Weekend\". Lennon began the recording of his own album, Mind Games, using the same players as on Feeling the Space, dubbed \"The Plastic U.F.Ono Band\". Around the time of the album's release in November, Lennon moved to Los Angeles with new lover May Pang. In October, Lennon began the recording of an album of rock 'n' roll oldies (a contractual obligation due to a lawsuit). These featured many Plastic Ono Band regulars (including much of the \"U.F.Ono Band\", Klaus Voorman, and the return of Phil Spector to the production chair), but upon release in 1975 as Rock 'n' Roll, it was credited to Lennon alone. The sessions for Rock 'n' Roll were extremely troubled, and the sessions were abandoned until a later date. In July 1974, Lennon returned to New York to record Walls and Bridges. The new \"Plastic Ono Nuclear Band\" featured both old and new faces, with Jim Keltner, Kenneth Ascher, and Arthur Jenkins continuing from Mind Games, the returns of Klaus Voorman, Nicky Hopkins, and Bobby Keys, and the addition of guitarists Jesse Ed Davis and Eddie Mottau. Recording was finished in August, and the album was released 26 September and 4 October in the US and UK respectively. Walls and Bridges would prove to be the last release of new material by the Plastic Ono Band in the 1970s. Lennon subsequently returned to his marriage with Ono and retired from music following the birth of his son Sean. The compilation Shaved Fish was released in October 1975, Lennon's last release credited to the Plastic Ono Band. Upon his and Ono's return to music in 1980 for the album Double Fantasy, they played with an all-new group of studio musicians who were not billed as any variation of the Plastic Ono Band name. Lennon was shot and killed shortly after the release of the album.", + "doc_id": "1" + }, + { + "text": " John Winston Ono Lennon (9 October 1940 - 8 December 1980) was an English singer, songwriter, and peace activist who co-founded the Beatles, the most commercially successful band in the history of popular music. He and fellow member Paul McCartney formed a much-celebrated songwriting partnership. Along with George Harrison and Ringo Starr, the group would ascend to world-wide fame during the 1960s. During his marriage to Cynthia, Lennon's first son Julian was born at the same time that his commitments with the Beatles were intensifying at the height of Beatlemania. Lennon was touring with the Beatles when Julian was born on 8 April 1963. Julian's birth, like his mother Cynthia's marriage to Lennon, was kept secret because Epstein was convinced that public knowledge of such things would threaten the Beatles' commercial success. Julian recalled that as a small child in Weybridge some four years later, \"I was trundled home from school and came walking up with one of my watercolour paintings. It was just a bunch of stars and this blonde girl I knew at school. And Dad said, 'What's this?' I said, 'It's Lucy in the sky with diamonds.'\" Lennon used it as the title of a Beatles song, and though it was later reported to have been derived from the initials LSD, Lennon insisted, \"It's not an acid song.\" McCartney corroborated Lennon's explanation that Julian innocently came up with the name. Lennon was distant from Julian, who felt closer to McCartney than to his father. During a car journey to visit Cynthia and Julian during Lennon's divorce, McCartney composed a song, \"Hey Jules\", to comfort him. It would evolve into the Beatles song \"Hey Jude\". Lennon later said, \"That's his best song. It started off as a song about my son Julian ... he turned it into 'Hey Jude'. I always thought it was about me and Yoko but he said it wasn't.\" Lennon's relationship with Julian was already strained, and after Lennon and Ono moved to Manhattan in 1971, Julian would not see his father again until 1973. With Pang's encouragement, arrangements were made for Julian (and his mother) to visit Lennon in Los Angeles, where they went to Disneyland. Julian started to see his father regularly, and Lennon gave him a drumming part on a Walls and Bridges track. He bought Julian a Gibson Les Paul guitar and other instruments, and encouraged his interest in music by demonstrating guitar chord techniques. Julian recalls that he and his father \"got on a great deal better\" during the time he spent in New York: \"We had a lot of fun, laughed a lot and had a great time in general.\" In a Playboy interview with David Sheff shortly before his death, Lennon said, \"Sean was a planned child, and therein lies the difference. I don't love Julian any less as a child. He's still my son, whether he came from a bottle of whiskey or because they didn't have pills in those days. He's here, he belongs to me, and he always will.\" He said he was trying to re-establish a connection with the then 17-year-old, and confidently predicted, \"Julian and I will have a relationship in the future.\" After his death it was revealed that he had left Julian very little in his will.", + "doc_id": "2" + } + ], + "structured_outputs": { + "json": { + "$defs": { + "_MODEL_OUTPUT_ENTRY": { + "properties": { + "r": { + "minimum": 0, + "title": "R", + "type": "integer" + }, + "c": { + "items": { + "minimum": 0, + "type": "integer" + }, + "title": "C", + "type": "array" + } + }, + "required": [ + "r", + "c" + ], + "title": "_MODEL_OUTPUT_ENTRY", + "type": "object" + } + }, + "items": { + "$ref": "#/$defs/_MODEL_OUTPUT_ENTRY" + }, + "title": "_MODEL_OUTPUT", + "type": "array" + } + } + }, + "temperature": 0.0, + "max_completion_tokens": 4096 +} \ No newline at end of file diff --git a/test/formatters/granite/testdata/test_canned_output/expected_result/context-attribution.json b/test/formatters/granite/testdata/test_canned_output/expected_result/context-attribution.json new file mode 100644 index 000000000..ea8b1ff91 --- /dev/null +++ b/test/formatters/granite/testdata/test_canned_output/expected_result/context-attribution.json @@ -0,0 +1,30 @@ +{ + "choices": [ + { + "index": 0, + "message": { + "content": "[{\"response_begin\": 0, \"response_end\": 137, \"response_text\": \"The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. \", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 0, \"response_end\": 137, \"response_text\": \"The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. \", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 249, \"attribution_end\": 411, \"attribution_text\": \"Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 249, \"attribution_end\": 411, \"attribution_text\": \"Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 130, \"attribution_end\": 249, \"attribution_text\": \"Lennon and Ono had begun a personal and artistic relationship in 1968, collaborating on several experimental releases. \"}]", + "role": "assistant", + "tool_calls": [], + "reasoning_content": null + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "prompt_logprobs": null, + "id": "chatcmpl-a7d7170784e84757b5bc365cb25c57c9", + "created": 1757963100, + "model": "context_attribution_all", + "object": "chat.completion", + "service_tier": null, + "system_fingerprint": null, + "usage": { + "completion_tokens": 32, + "prompt_tokens": 1124, + "total_tokens": 1156, + "completion_tokens_details": null, + "prompt_tokens_details": null + }, + "kv_transfer_params": null +} diff --git a/test/formatters/granite/testdata/test_canned_output/model_output/context-attribution.json b/test/formatters/granite/testdata/test_canned_output/model_output/context-attribution.json new file mode 100644 index 000000000..def7c8a90 --- /dev/null +++ b/test/formatters/granite/testdata/test_canned_output/model_output/context-attribution.json @@ -0,0 +1,35 @@ +{ + "id": "chatcmpl-a7d7170784e84757b5bc365cb25c57c9", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "[{\"r\": 0, \"c\": [0, 2]}, {\"r\": 1, \"c\": [2, 0, 1]}]", + "refusal": null, + "role": "assistant", + "annotations": null, + "audio": null, + "function_call": null, + "tool_calls": [], + "reasoning_content": null + }, + "stop_reason": null + } + ], + "created": 1757963100, + "model": "context_attribution_all", + "object": "chat.completion", + "service_tier": null, + "system_fingerprint": null, + "usage": { + "completion_tokens": 32, + "prompt_tokens": 1124, + "total_tokens": 1156, + "completion_tokens_details": null, + "prompt_tokens_details": null + }, + "prompt_logprobs": null, + "kv_transfer_params": null +} diff --git a/test/formatters/granite/testdata/test_run_ollama/context-attribution.json b/test/formatters/granite/testdata/test_run_ollama/context-attribution.json new file mode 100644 index 000000000..47a3c9267 --- /dev/null +++ b/test/formatters/granite/testdata/test_run_ollama/context-attribution.json @@ -0,0 +1,11 @@ +{ + "choices": [ + { + "index": 0, + "message": { + "content": "[{\"response_begin\": 0, \"response_end\": 137, \"response_text\": \"The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. \", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 0, \"response_end\": 137, \"response_text\": \"The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. \", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 249, \"attribution_end\": 411, \"attribution_text\": \"Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 249, \"attribution_end\": 411, \"attribution_text\": \"Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 130, \"attribution_end\": 249, \"attribution_text\": \"Lennon and Ono had begun a personal and artistic relationship in 1968, collaborating on several experimental releases. \"}]", + "role": "assistant" + } + } + ] +} diff --git a/test/formatters/granite/testdata/test_run_transformers/context-attribution.json b/test/formatters/granite/testdata/test_run_transformers/context-attribution.json new file mode 100644 index 000000000..4693c5801 --- /dev/null +++ b/test/formatters/granite/testdata/test_run_transformers/context-attribution.json @@ -0,0 +1,11 @@ +{ + "choices": [ + { + "index": 0, + "message": { + "content": "[{\"response_begin\": 0, \"response_end\": 137, \"response_text\": \"The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. \", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 0, \"response_end\": 137, \"response_text\": \"The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. \", \"attribution_doc_id\": \"1\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 249, \"attribution_end\": 411, \"attribution_text\": \"Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 130, \"attribution_end\": 249, \"attribution_text\": \"Lennon and Ono had begun a personal and artistic relationship in 1968, collaborating on several experimental releases. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"1\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 411, \"attribution_end\": 661, \"attribution_text\": \"The band would go on to feature a rotating lineup of many musicians, including Eric Clapton, Klaus Voormann, Alan White, Billy Preston, Jim Keltner, Delaney & Bonnie and Friends, and Lennon's former Beatles bandmates George Harrison and Ringo Starr. \"}]", + "role": "assistant" + } + } + ] +}