Feat/music annotation (#18391)

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

web/app/components/base/markdown-blocks/music.tsx

@@ -0,0 +1,36 @@
+import abcjs from 'abcjs'
+import { useEffect, useRef } from 'react'
+import 'abcjs/abcjs-audio.css'
+
+const MarkdownMusic = ({ children }: { children: React.ReactNode }) => {
+  const containerRef = useRef<HTMLDivElement>(null)
+  const controlsRef = useRef<HTMLDivElement>(null)
+
+  useEffect(() => {
+    if (containerRef.current && controlsRef.current) {
+      if (typeof children === 'string') {
+        const visualObjs = abcjs.renderAbc(containerRef.current, children)
+        const synthControl = new abcjs.synth.SynthController()
+        synthControl.load(controlsRef.current, {}, { displayPlay: true })
+        const synth = new abcjs.synth.CreateSynth()
+        const visualObj = visualObjs[0]
+        synth.init({ visualObj }).then(() => {
+          synthControl.setTune(visualObj, false)
+        })
+        containerRef.current.style.overflow = 'auto'
+      }
+    }
+  }, [children])
+
+  return (
+    <div style={{ minHeight: '350px', minWidth: '100%', overflow: 'auto' }}>
+      <div ref={containerRef} />
+      <div
+        ref={controlsRef}
+      />
+    </div>
+  )
+}
+MarkdownMusic.displayName = 'MarkdownMusic'
+
+export default MarkdownMusic

web/app/components/base/markdown.tsx

@@ -23,6 +23,7 @@ import VideoGallery from '@/app/components/base/video-gallery'
 import AudioGallery from '@/app/components/base/audio-gallery'
 import MarkdownButton from '@/app/components/base/markdown-blocks/button'
 import MarkdownForm from '@/app/components/base/markdown-blocks/form'
+import MarkdownMusic from '@/app/components/base/markdown-blocks/music'
 import ThinkBlock from '@/app/components/base/markdown-blocks/think-block'
 import { Theme } from '@/types/app'
 import useTheme from '@/hooks/use-theme'
@@ -51,6 +52,7 @@ const capitalizationLanguageNameMap: Record<string, string> = {
   json: 'JSON',
   latex: 'Latex',
   svg: 'SVG',
+  abc: 'ABC',
 }
 const getCorrectCapitalizationLanguageName = (language: string) => {
   if (!language)
@@ -137,45 +139,54 @@ const CodeBlock: any = memo(({ inline, className, children, ...props }: any) =>
 
   const renderCodeContent = useMemo(() => {
     const content = String(children).replace(/\n$/, '')
-    if (language === 'mermaid' && isSVG) {
-      return <Flowchart PrimitiveCode={content} />
-    }
-    else if (language === 'echarts') {
-      return (
-        <div style={{ minHeight: '350px', minWidth: '100%', overflowX: 'scroll' }}>
+    switch (language) {
+      case 'mermaid':
+        if (isSVG)
+          return <Flowchart PrimitiveCode={content} />
+        break
+      case 'echarts':
+        return (
+          <div style={{ minHeight: '350px', minWidth: '100%', overflowX: 'scroll' }}>
+            <ErrorBoundary>
+              <ReactEcharts option={chartData} style={{ minWidth: '700px' }} />
+            </ErrorBoundary>
+          </div>
+        )
+      case 'svg':
+        if (isSVG) {
+          return (
+            <ErrorBoundary>
+              <SVGRenderer content={content} />
+            </ErrorBoundary>
+          )
+        }
+        break
+      case 'abc':
+        return (
           <ErrorBoundary>
-            <ReactEcharts option={chartData} style={{ minWidth: '700px' }} />
+            <MarkdownMusic children={content} />
           </ErrorBoundary>
-        </div>
-      )
+        )
+      default:
+        return (
+          <SyntaxHighlighter
+            {...props}
+            style={theme === Theme.light ? atelierHeathLight : atelierHeathDark}
+            customStyle={{
+              paddingLeft: 12,
+              borderBottomLeftRadius: '10px',
+              borderBottomRightRadius: '10px',
+              backgroundColor: 'var(--color-components-input-bg-normal)',
+            }}
+            language={match?.[1]}
+            showLineNumbers
+            PreTag="div"
+          >
+            {content}
+          </SyntaxHighlighter>
+        )
     }
-    else if (language === 'svg' && isSVG) {
-      return (
-        <ErrorBoundary>
-          <SVGRenderer content={content} />
-        </ErrorBoundary>
-      )
-    }
-    else {
-      return (
-        <SyntaxHighlighter
-          {...props}
-          style={theme === Theme.light ? atelierHeathLight : atelierHeathDark}
-          customStyle={{
-            paddingLeft: 12,
-            borderBottomLeftRadius: '10px',
-            borderBottomRightRadius: '10px',
-            backgroundColor: 'var(--color-components-input-bg-normal)',
-          }}
-          language={match?.[1]}
-          showLineNumbers
-          PreTag="div"
-        >
-          {content}
-        </SyntaxHighlighter>
-      )
-    }
-  }, [language, match, props, children, chartData, isSVG])
+  }, [children, language, isSVG, chartData, props, theme, match])
 
   if (inline || !match)
     return <code {...props} className={className}>{children}</code>

web/app/styles/markdown.scss

@@ -1039,3 +1039,6 @@
 .markdown-body .react-syntax-highlighter-line-number {
   color: var(--color-text-quaternary);
 }
+.markdown-body .abcjs-inline-audio .abcjs-btn {
+  display: flex !important;
+}

web/package.json

@@ -57,6 +57,7 @@
     "@tanstack/react-form": "^1.3.3",
     "@tanstack/react-query": "^5.60.5",
     "@tanstack/react-query-devtools": "^5.60.5",
+    "abcjs": "^6.4.4",
     "ahooks": "^3.8.4",
     "class-variance-authority": "^0.7.0",
     "classnames": "^2.5.1",

web/pnpm-lock.yaml

@@ -103,6 +103,9 @@ importers:
       '@tanstack/react-query-devtools':
         specifier: ^5.60.5
         version: 5.72.2(@tanstack/react-query@5.72.2(react@19.0.0))(react@19.0.0)
+      abcjs:
+        specifier: ^6.4.4
+        version: 6.4.4
       ahooks:
         specifier: ^3.8.4
         version: 3.8.4(react@19.0.0)
@@ -3416,6 +3419,9 @@ packages:
   abbrev@1.1.1:
     resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==}
 
+  abcjs@6.4.4:
+    resolution: {integrity: sha512-dT3Z2vb8yihbiPMzSoup0JOcvO2je4qpFNlTD+kS5VBelE3AASAs18dS5qeMWkZeqCz7kI/hz62B2lpMDugWLA==}
+
   abort-controller@3.0.0:
     resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==}
     engines: {node: '>=6.5'}
@@ -12127,6 +12133,8 @@ snapshots:
   abbrev@1.1.1:
     optional: true
 
+  abcjs@6.4.4: {}
+
   abort-controller@3.0.0:
     dependencies:
       event-target-shim: 5.0.1