Citations UI at the bottom (no summary) (#2009)

* Citations UI at the bottom (no summary)

Related issue #1990

Notes

- No ArrowOutofBox icon => used LinkIcon as a placeholder
- Citations' appearances is streamed (a citation appear when the citation link appears)
- Did as per figma with overflowing citations and hidden scroll
  - question mark on intuitivity of hidden scroll for users (shift+scroll is not natural for everybody)
  - => should we get it to switch to natural horizontal scroll on hover on citations (without shift)? spent 5mn to look how to do it but apparently this is [non-standard](tailwindlabs/tailwindcss#7869). it could still be done using a pure JS solution, WDYT?

* overflow AND scroll, right AND left

* rebase

* on citation hover, scroll to its footer definition

* mobile changes since rebase - fix

* spolu review

* spolu review better

front/components/assistant/RenderMessageMarkdown.tsx

@@ -6,7 +6,7 @@ import {
   Tooltip,
 } from "@dust-tt/sparkle";
 import dynamic from "next/dynamic";
-import React, { useState } from "react";
+import React, { useEffect, useState } from "react";
 import ReactMarkdown from "react-markdown";
 import { ReactMarkdownProps } from "react-markdown/lib/complex-types";
 import remarkDirective from "remark-directive";
@@ -121,24 +121,42 @@ function addClosingBackticks(str: string): string {
   return str;
 }
 
-export const ReferencesContext = React.createContext<{
-  [key: string]: RetrievalDocumentType;
-}>({});
+type CitationsContextType = {
+  references: {
+    [key: string]: RetrievalDocumentType;
+  };
+  updateActiveReferences: (doc: RetrievalDocumentType, index: number) => void;
+  setHoveredReference: (index: number | null) => void;
+};
+
+export const CitationsContext = React.createContext<CitationsContextType>({
+  references: {},
+  updateActiveReferences: () => null,
+  setHoveredReference: () => null,
+});
 
 export function RenderMessageMarkdown({
   content,
   blinkingCursor,
-  references,
   agentConfigurations,
+  citationsContext,
 }: {
   content: string;
   blinkingCursor: boolean;
-  references?: { [key: string]: RetrievalDocumentType };
   agentConfigurations?: AgentConfigurationType[];
+  citationsContext?: CitationsContextType;
 }) {
   return (
     <div className={blinkingCursor ? "blinking-cursor" : ""}>
-      <ReferencesContext.Provider value={references || {}}>
+      <CitationsContext.Provider
+        value={
+          citationsContext || {
+            references: {},
+            updateActiveReferences: () => null,
+            setHoveredReference: () => null,
+          }
+        }
+      >
         <ReactMarkdown
           linkTarget="_blank"
           components={{
@@ -178,7 +196,7 @@ export function RenderMessageMarkdown({
         >
           {addClosingBackticks(content)}
         </ReactMarkdown>
-      </ReferencesContext.Provider>
+      </CitationsContext.Provider>
     </div>
   );
 }
@@ -213,21 +231,32 @@ function isCiteProps(props: ReactMarkdownProps): props is ReactMarkdownProps & {
 }
 
 function CiteBlock(props: ReactMarkdownProps) {
-  const references = React.useContext(ReferencesContext);
-
-  if (isCiteProps(props) && props.references) {
-    const refs = (
-      JSON.parse(props.references) as {
-        counter: number;
-        ref: string;
-      }[]
-    ).filter((r) => r.ref in references);
+  const { references, updateActiveReferences, setHoveredReference } =
+    React.useContext(CitationsContext);
+  const refs =
+    isCiteProps(props) && props.references
+      ? (
+          JSON.parse(props.references) as {
+            counter: number;
+            ref: string;
+          }[]
+        ).filter((r) => r.ref in references)
+      : undefined;
+
+  useEffect(() => {
+    if (refs) {
+      refs.forEach((r) => {
+        const document = references[r.ref];
+        updateActiveReferences(document, r.counter);
+      });
+    }
+  }, [refs, references, updateActiveReferences]);
 
+  if (refs) {
     return (
       <>
         {refs.map((r, i) => {
           const document = references[r.ref];
-
           const provider = providerFromDocument(document);
           const title = titleFromDocument(document);
           const link = linkFromDocument(document);
@@ -263,6 +292,7 @@ function CiteBlock(props: ReactMarkdownProps) {
                   target="_blank"
                   rel="noopener noreferrer"
                   className={citeClassNames}
+                  onMouseEnter={() => setHoveredReference(r.counter)}
                 >
                   {r.counter}
                 </a>

front/components/assistant/conversation/AgentMessage.tsx

@@ -4,15 +4,25 @@ import {
   Chip,
   ClipboardIcon,
   DocumentDuplicateIcon,
+  DocumentTextIcon,
   DropdownMenu,
+  ExternalLinkIcon,
   EyeIcon,
+  IconButton,
   Spinner,
 } from "@dust-tt/sparkle";
-import { useCallback, useContext, useEffect, useState } from "react";
+import Link from "next/link";
+import { useCallback, useContext, useEffect, useRef, useState } from "react";
 
 import { AgentAction } from "@app/components/assistant/conversation/AgentAction";
 import { ConversationMessage } from "@app/components/assistant/conversation/ConversationMessage";
 import { GenerationContext } from "@app/components/assistant/conversation/GenerationContextProvider";
+import {
+  linkFromDocument,
+  PROVIDER_LOGO_PATH,
+  providerFromDocument,
+  titleFromDocument,
+} from "@app/components/assistant/conversation/RetrievalAction";
 import { RenderMessageMarkdown } from "@app/components/assistant/RenderMessageMarkdown";
 import { useEventSource } from "@app/hooks/useEventSource";
 import {
@@ -24,6 +34,7 @@ import {
   AgentMessageSuccessEvent,
 } from "@app/lib/api/assistant/agent";
 import { GenerationTokensEvent } from "@app/lib/api/assistant/generation";
+import { classNames } from "@app/lib/utils";
 import {
   isRetrievalActionType,
   RetrievalDocumentType,
@@ -224,7 +235,21 @@ export function AgentMessage({
   const [references, setReferences] = useState<{
     [key: string]: RetrievalDocumentType;
   }>({});
-
+  const [activeReferences, setActiveReferences] = useState<
+    { index: number; document: RetrievalDocumentType }[]
+  >([]);
+  function updateActiveReferences(
+    document: RetrievalDocumentType,
+    index: number
+  ) {
+    const existingIndex = activeReferences.find((r) => r.index === index);
+    if (!existingIndex) {
+      setActiveReferences([...activeReferences, { index, document }]);
+    }
+  }
+  const [lastHoveredReference, setLastHoveredReference] = useState<
+    number | null
+  >(null);
   useEffect(() => {
     if (
       agentMessageToRender.action &&
@@ -311,7 +336,15 @@ export function AgentMessage({
             <RenderMessageMarkdown
               content={agentMessage.content}
               blinkingCursor={streaming}
-              references={references}
+              citationsContext={{
+                references,
+                updateActiveReferences,
+                setHoveredReference: setLastHoveredReference,
+              }}
+            />
+            <Citations
+              activeReferences={activeReferences}
+              lastHoveredReference={lastHoveredReference}
             />
           </div>
         )}
@@ -339,6 +372,95 @@ export function AgentMessage({
   }
 }
 
+function Citations({
+  activeReferences,
+  lastHoveredReference,
+}: {
+  activeReferences: { index: number; document: RetrievalDocumentType }[];
+  lastHoveredReference: number | null;
+}) {
+  const citationContainer = useRef<HTMLDivElement>(null);
+
+  useEffect(() => {
+    if (citationContainer.current) {
+      if (lastHoveredReference !== null) {
+        citationContainer.current.scrollTo({
+          left: citationsScrollOffset(lastHoveredReference),
+          behavior: "smooth",
+        });
+      }
+    }
+  }, [lastHoveredReference]);
+
+  function citationsScrollOffset(reference: number | null) {
+    if (!citationContainer.current || reference === null) {
+      return 0;
+    }
+    const offset = (
+      citationContainer.current.firstElementChild
+        ?.firstElementChild as HTMLElement
+    ).offsetLeft;
+    const scrolling =
+      (citationContainer.current.firstElementChild?.firstElementChild
+        ?.scrollWidth || 0) *
+      (reference - 2);
+    return scrolling - offset;
+  }
+
+  activeReferences.sort((a, b) => a.index - b.index);
+  return (
+    <div
+      className="-mx-[100%] mt-9 overflow-x-auto px-[100%] scrollbar-hide"
+      ref={citationContainer}
+    >
+      <div className="left-100 relative flex gap-2">
+        {activeReferences.map(({ document, index }) => {
+          const provider = providerFromDocument(document);
+          return (
+            <div
+              className={classNames(
+                "flex w-48 flex-none flex-col gap-2 rounded-xl border border-structure-100 p-3 sm:w-64",
+                lastHoveredReference === index
+                  ? "animate-[bgblink_500ms_3]"
+                  : ""
+              )}
+              key={index}
+            >
+              <div className="flex items-center gap-1.5">
+                <div className="flex h-5 w-5 items-center justify-center rounded-full border border-violet-200 bg-violet-100 text-xs font-semibold text-element-800">
+                  {index}
+                </div>
+                <div className="h-5 w-5">
+                  {provider === "none" ? (
+                    <DocumentTextIcon className="h-5 w-5 text-slate-500" />
+                  ) : (
+                    <img
+                      src={PROVIDER_LOGO_PATH[providerFromDocument(document)]}
+                      className="h-5 w-5"
+                    />
+                  )}
+                </div>
+                <div className="flex-grow text-xs" />
+                <Link href={linkFromDocument(document)} target="_blank">
+                  <IconButton
+                    icon={ExternalLinkIcon}
+                    size="xs"
+                    variant="primary"
+                  />
+                </Link>
+              </div>
+              <div className="text-xs font-bold text-element-900">
+                {titleFromDocument(document)}
+              </div>
+            </div>
+          );
+        })}
+        <div className="h-1 w-[100%] flex-none" />
+      </div>
+    </div>
+  );
+}
+
 function ErrorMessage({
   error,
   retryHandler,

front/components/assistant/conversation/ConversationMessage.tsx

@@ -115,7 +115,7 @@ export function ConversationMessage({
     <>
       {/* SMALL SIZE SCREEN*/}
       <div className="flex w-full gap-4 xl:hidden">
-        <div className="flex flex-grow flex-col gap-4">
+        <div className="flex w-full flex-grow flex-col gap-4">
           <div className="flex items-start gap-2">
             <div className="flex h-8 flex-grow items-center gap-2">
               <Avatar
@@ -239,7 +239,9 @@ export function ConversationMessage({
           busy={avatarBusy}
         />
 
-        {/* COLUMN 2: CONTENT */}
+        {/* COLUMN 2: CONTENT
+         * min-w-0 prevents the content from overflowing the container
+         */}
         <div className="flex min-w-0 flex-grow flex-col gap-4">
           <div className="text-sm font-medium">{name}</div>
           <div className="min-w-0 break-words text-base font-normal">

front/components/sparkle/AppLayout.tsx

@@ -361,7 +361,7 @@ export default function AppLayout({
           >
             <div
               className={classNames(
-                "mx-auto h-full",
+                "mx-auto h-full overflow-x-hidden",
                 isWideMode ? "w-full" : "max-w-4xl px-6"
               )}
             >

front/styles/global.css

@@ -67,3 +67,13 @@ main {
     transform: translate3d(8px, 3px, 0);
   }
 }
+
+@keyframes bgblink {
+  0%,
+  100% {
+    @apply bg-structure-0;
+  }
+  50% {
+    @apply bg-structure-200;
+  }
+}