From e5e857b0876622a98a1c1270e1896864ec1e8a1a Mon Sep 17 00:00:00 2001
From: Talon <talon@iamtalon.me>
Date: Thu, 12 Jun 2025 20:45:51 +0200
Subject: [PATCH] Try to fix slowdown when rendering large amount of output

---
 src/lib/components/MudTerminal.svelte | 156 +++++------------------
 src/lib/stores/mudStore.ts            | 148 +++++++++++++++++++---
 src/lib/utils/textProcessing.ts       | 175 ++++++++++++++++++++++++++
 3 files changed, 338 insertions(+), 141 deletions(-)
 create mode 100644 src/lib/utils/textProcessing.ts

diff --git a/src/lib/components/MudTerminal.svelte b/src/lib/components/MudTerminal.svelte
index a86d27c..f7f86d3 100644
--- a/src/lib/components/MudTerminal.svelte
+++ b/src/lib/components/MudTerminal.svelte
@@ -1,12 +1,11 @@
 <script lang="ts">
   import { onMount, onDestroy, createEventDispatcher } from 'svelte';
-  import { activeOutputHistory, addToOutputHistory, addToInputHistory, navigateInputHistory, activeInputHistoryIndex, activeConnection, uiSettings, accessibilitySettings, activeInputHistory, activeProfileId, connectionStatus } from '$lib/stores/mudStore';
+  import { activeRenderableLines, addToOutputHistory, addToInputHistory, navigateInputHistory, activeInputHistoryIndex, activeConnection, uiSettings, accessibilitySettings, activeInputHistory, activeProfileId, connectionStatus } from '$lib/stores/mudStore';
   import { tick } from 'svelte';
-  import AnsiToHtml from 'ansi-to-html';
   import { AccessibilityManager } from '$lib/accessibility/AccessibilityManager';
 
   // Create safe defaults for reactivity
-  $: safeOutputHistory = $activeOutputHistory || [];
+  $: safeRenderableLines = $activeRenderableLines || [];
   $: safeActiveProfileId = $activeProfileId || null;
 
   // Create event dispatcher
@@ -22,79 +21,11 @@
   let inputElement: HTMLInputElement;
   let currentInput = '';
   let accessibilityManager: AccessibilityManager | null = null;
-  let ansiConverter = new AnsiToHtml({
-    fg: '#f8f8f2',
-    bg: '#282a36',
-    newline: false, // We'll handle newlines ourselves
-    escapeXML: true,
-    stream: false
-  });
   
   // Message navigation state
   let currentFocusedMessageIndex: number = -1;
   let messageElements: HTMLElement[] = [];
   
-  // Process ANSI color codes
-  function processAnsi(text: string): string {
-    if ($uiSettings.ansiColor) {
-      try {
-        // First process ANSI to HTML without replacing newlines
-        const ansiProcessed = ansiConverter.toHtml(text);
-        
-        // Then replace newlines with <br> tags
-        return ansiProcessed.replace(/\r\n|\r|\n/g, '<br>');
-      } catch (error) {
-        console.error('Error processing ANSI colors:', error);
-        // Fallback to just replacing newlines
-        return text.replace(/\r\n|\r|\n/g, '<br>');
-      }
-    } else {
-      // Strip ANSI codes if color is disabled
-      return text.replace(/\u001b\[\d+(;\d+)*m/g, '')
-                .replace(/\r\n|\r|\n/g, '<br>');
-    }
-  }
-  
-  // Split text into individual lines (for better screen reader navigation)
-  function splitIntoLines(text: string): string[] {
-    // First handle any text that already has <br> tags from ANSI processing
-    if (text.includes('<br>')) {
-      return text.split('<br>').filter(line => line.trim().length > 0);
-    }
-    
-    // Otherwise split by newlines
-    return text.split(/\r\n|\r|\n/).filter(line => line.trim().length > 0);
-  }
-  
-  // Apply highlighting to text
-  function applyHighlights(text: string, highlights: { pattern: string; color: string; isRegex: boolean }[]): string {
-    if (!highlights || highlights.length === 0) return text;
-    
-    let highlightedText = text;
-    
-    highlights.forEach(({ pattern, color, isRegex }) => {
-      if (isRegex) {
-        try {
-          const regex = new RegExp(pattern, 'g');
-          highlightedText = highlightedText.replace(regex, (match) => {
-            return `<span style="background-color: ${color};">${match}</span>`;
-          });
-        } catch (error) {
-          console.error('Invalid regex pattern:', pattern, error);
-        }
-      } else {
-        // Escape special characters in the pattern for use in a regex
-        const safePattern = pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-        const regex = new RegExp(safePattern, 'g');
-        highlightedText = highlightedText.replace(regex, (match) => {
-          return `<span style="background-color: ${color};">${match}</span>`;
-        });
-      }
-    });
-    
-    return highlightedText;
-  }
-  
   // Handle input submission
   async function handleSubmit(event: Event) {
     event.preventDefault();
@@ -288,22 +219,20 @@
     }
   }
   
-  // Watch output history and profile changes to update display
-  $: {
-    if (safeOutputHistory.length > 0) {
-      console.log('Active output history changed, updating terminal display');
+  // Watch for renderable lines changes to update display - optimized reactive statement
+  $: if (safeRenderableLines.length > 0) {
+    // Only scroll and update elements when new lines are added
+    // Use a microtask to batch DOM updates
+    Promise.resolve().then(() => {
       scrollToBottom();
-      // Update message elements when output history changes
-      setTimeout(updateMessageElements, 0);
-    }
+      updateMessageElements();
+    });
   }
   
-  $: {
-    // Every time the active profile changes, update the terminal content
-    if ($activeProfileId) {
-      console.log(`Active profile is now: ${$activeProfileId}, updating output display`);
-      updateOutputDisplay();
-    }
+  // Watch for active profile changes
+  $: if ($activeProfileId) {
+    console.log(`Active profile is now: ${$activeProfileId}, updating output display`);
+    updateOutputDisplay();
   }
   
   // Function to update the displayed output based on the active profile
@@ -359,47 +288,20 @@
        tabindex="0"
        on:keydown={handleOutputKeyDown}
        style="font-family: {$uiSettings.font}; font-size: {$accessibilitySettings.fontSize}px; line-height: {$accessibilitySettings.lineSpacing};">
-    {#each safeOutputHistory as item (item.id)}
-      <!-- For input lines, keep them as a single block -->
-      {#if item.isInput}
-        <div class="mud-terminal-line mud-input-line" tabindex="-1">
-          {#if $uiSettings.showTimestamps}
-            <span class="mud-timestamp" aria-hidden="true">[{formatTimestamp(item.timestamp)}]</span>
-          {/if}
-          <div class="mud-terminal-content">
-            {@html item.text}
-          </div>
-        </div>
-      <!-- For MUD output, split into individual navigable lines -->
-      {:else}
-        <!-- Process the content first -->
-        {@const processedContent = applyHighlights(processAnsi(item.text), item.highlights || [])}
-        {@const lines = splitIntoLines(processedContent)}
-        
-        <!-- If no lines or only one line, render as is -->
-        {#if lines.length <= 1}
-          <div class="mud-terminal-line" tabindex="-1">
-            {#if $uiSettings.showTimestamps}
-              <span class="mud-timestamp" aria-hidden="true">[{formatTimestamp(item.timestamp)}]</span>
-            {/if}
-            <div class="mud-terminal-content">
-              {@html processedContent}
-            </div>
-          </div>
-        <!-- Otherwise render each line separately for better navigation -->
-        {:else}
-          {#each lines as line, lineIndex}
-            <div class="mud-terminal-line mud-terminal-subline" tabindex="-1">
-              {#if $uiSettings.showTimestamps && lineIndex === 0}
-                <span class="mud-timestamp" aria-hidden="true">[{formatTimestamp(item.timestamp)}]</span>
-              {/if}
-              <div class="mud-terminal-content">
-                {@html line}
-              </div>
-            </div>
-          {/each}
+    
+    <!-- Optimized rendering using pre-processed lines -->
+    {#each safeRenderableLines as line (line.id)}
+      <div class="mud-terminal-line" 
+           class:mud-input-line={line.isInput} 
+           class:mud-terminal-subline={line.isSubline} 
+           tabindex="-1">
+        {#if $uiSettings.showTimestamps && line.lineIndex === 0}
+          <span class="mud-timestamp" aria-hidden="true">[{formatTimestamp(line.timestamp)}]</span>
         {/if}
-      {/if}
+        <div class="mud-terminal-content">
+          {@html line.content}
+        </div>
+      </div>
     {/each}
   </div>
   
@@ -593,16 +495,16 @@
     border-width: 0;
   }
   
-  .focused-message {
+  :global(.focused-message) {
     outline: 2px solid var(--color-primary, #2196f3);
     background-color: rgba(33, 150, 243, 0.1);
   }
   
-  .dark-mode .focused-message {
+  .dark-mode :global(.focused-message) {
     background-color: rgba(255, 121, 198, 0.2); /* Use a pink highlight for dark mode */
   }
   
-  .high-contrast .focused-message {
+  .high-contrast :global(.focused-message) {
     outline: 3px solid #fff;
     background-color: #444;
   }
diff --git a/src/lib/stores/mudStore.ts b/src/lib/stores/mudStore.ts
index 8303159..1aac5ca 100644
--- a/src/lib/stores/mudStore.ts
+++ b/src/lib/stores/mudStore.ts
@@ -3,6 +3,7 @@ import { settingsManager } from '$lib/settings/SettingsManager';
 import type { MudProfile } from '$lib/profiles/ProfileManager';
 import type { MudConnection } from '$lib/connection/MudConnection';
 import type { Trigger } from '$lib/triggers/TriggerSystem';
+import { processMessage, createCacheKey, type ProcessedMessage } from '$lib/utils/textProcessing';
 
 // Store for active connections
 export const connections = writable<{ [key: string]: MudConnection }>({});
@@ -27,6 +28,11 @@ export const outputHistory = writable<{
   }[];
 }>({});
 
+// Store for processed output history - keyed by profile ID
+export const processedOutputHistory = writable<{ 
+  [profileId: string]: ProcessedMessage[];
+}>({});
+
 // Store for connection status
 export const connectionStatus = writable<{ [key: string]: 'connected' | 'disconnected' | 'connecting' | 'error' }>({});
 
@@ -90,6 +96,93 @@ export const activeOutputHistory = derived(
   }
 );
 
+// Derived store for active processed output history with UI settings awareness
+export const activeProcessedOutputHistory = derived(
+  [processedOutputHistory, activeProfileId, uiSettings],
+  ([$processedOutputHistory, $activeProfileId, $uiSettings]) => {
+    const profileId = $activeProfileId || 'default';
+    const messages = $processedOutputHistory[profileId] || [];
+    
+    // Re-cache messages if UI settings that affect processing have changed
+    const cacheKey = createCacheKey($uiSettings.ansiColor);
+    
+    return messages.map(message => {
+      // Check if we need to update the cache for current settings
+      if (!message.processedCache.has(cacheKey)) {
+        // Re-process with current settings
+        const reprocessed = processMessage({
+          id: message.id,
+          text: message.originalText,
+          timestamp: message.timestamp,
+          isInput: message.isInput,
+          highlights: message.highlights
+        }, $uiSettings.ansiColor);
+        
+        // Update the cache
+        message.processedCache.set(cacheKey, {
+          content: reprocessed.processedContent,
+          lines: reprocessed.lines
+        });
+        message.processedContent = reprocessed.processedContent;
+        message.lines = reprocessed.lines;
+      } else {
+        // Use cached version
+        const cached = message.processedCache.get(cacheKey)!;
+        message.processedContent = cached.content;
+        message.lines = cached.lines;
+      }
+      
+      return message;
+    });
+  }
+);
+
+// Derived store for flattened renderable lines - optimized for terminal display
+export const activeRenderableLines = derived(
+  [activeProcessedOutputHistory],
+  ([$activeProcessedOutputHistory]) => {
+    const lines: Array<{
+      id: string;
+      messageId: string;
+      content: string;
+      timestamp: number;
+      isInput: boolean;
+      isSubline: boolean;
+      lineIndex: number;
+    }> = [];
+
+    $activeProcessedOutputHistory.forEach(message => {
+      if (message.isInput) {
+        // Input messages are always single lines
+        lines.push({
+          id: message.id,
+          messageId: message.id,
+          content: message.processedContent,
+          timestamp: message.timestamp,
+          isInput: true,
+          isSubline: false,
+          lineIndex: 0
+        });
+      } else {
+        // Output messages may have multiple lines
+        message.lines.forEach(line => {
+          lines.push({
+            id: line.id,
+            messageId: message.id,
+            content: line.content,
+            timestamp: message.timestamp,
+            isInput: false,
+            isSubline: line.isSubline,
+            lineIndex: line.lineIndex
+          });
+        });
+      }
+    });
+
+    return lines;
+  }
+);
+
 // Derived store for active input history
 export const activeInputHistory = derived(
   [inputHistory, activeProfileId],
@@ -123,25 +216,25 @@ export const gmcpDebugLog = writable<{ id: string; module: string; data: any; ti
 // Helper functions
 export function addToOutputHistory(text: string, isInput = false, highlights: { pattern: string; color: string; isRegex: boolean }[] = []) {
   const profileId = get(activeProfileId);
+  const targetProfileId = profileId || 'default';
+  const maxSize = get(uiSettings).outputBufferSize;
+  const currentAnsiSetting = get(uiSettings).ansiColor;
   
+  const newItem = {
+    id: `output-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`,
+    text,
+    timestamp: Date.now(),
+    isInput,
+    highlights
+  };
+
+  // Update raw output history
   outputHistory.update(allHistory => {
-    // Default profile ID for cases where there's no active profile
-    const targetProfileId = profileId || 'default';
-    
     // Initialize history for this profile if it doesn't exist
     if (!allHistory[targetProfileId]) {
       allHistory[targetProfileId] = [];
     }
-    
-    const maxSize = get(uiSettings).outputBufferSize;
-    const newItem = {
-      id: `output-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`,
-      text,
-      timestamp: Date.now(),
-      isInput,
-      highlights
-    };
-    
+
     // Limit history size for this profile
     const updatedHistory = [...allHistory[targetProfileId], newItem];
     if (updatedHistory.length > maxSize) {
@@ -149,9 +242,30 @@ export function addToOutputHistory(text: string, isInput = false, highlights: {
     } else {
       allHistory[targetProfileId] = updatedHistory;
     }
-    
+
     return allHistory;
   });
+
+  // Update processed output history
+  processedOutputHistory.update(allProcessedHistory => {
+    // Initialize processed history for this profile if it doesn't exist
+    if (!allProcessedHistory[targetProfileId]) {
+      allProcessedHistory[targetProfileId] = [];
+    }
+
+    // Process the new message
+    const processedMessage = processMessage(newItem, currentAnsiSetting);
+
+    // Limit processed history size for this profile
+    const updatedProcessedHistory = [...allProcessedHistory[targetProfileId], processedMessage];
+    if (updatedProcessedHistory.length > maxSize) {
+      allProcessedHistory[targetProfileId] = updatedProcessedHistory.slice(updatedProcessedHistory.length - maxSize);
+    } else {
+      allProcessedHistory[targetProfileId] = updatedProcessedHistory;
+    }
+
+    return allProcessedHistory;
+  });
 }
 
 /**
@@ -293,6 +407,12 @@ export function clearOutputHistory() {
     allHistory[targetProfileId] = [];
     return allHistory;
   });
+
+  processedOutputHistory.update(allProcessedHistory => {
+    // Clear only the current profile's processed history
+    allProcessedHistory[targetProfileId] = [];
+    return allProcessedHistory;
+  });
 }
 
 export function updateGmcpData(module: string, data: any) {
diff --git a/src/lib/utils/textProcessing.ts b/src/lib/utils/textProcessing.ts
new file mode 100644
index 0000000..48ad5db
--- /dev/null
+++ b/src/lib/utils/textProcessing.ts
@@ -0,0 +1,175 @@
+import AnsiToHtml from 'ansi-to-html';
+
+// Create a singleton instance of the ANSI converter for consistent processing
+const ansiConverter = new AnsiToHtml({
+  fg: '#f8f8f2',
+  bg: '#282a36',
+  newline: false, // We'll handle newlines ourselves
+  escapeXML: true,
+  stream: false
+});
+
+export interface ProcessedLine {
+  id: string;
+  content: string;
+  isSubline: boolean;
+  parentId: string;
+  lineIndex: number;
+}
+
+export interface ProcessedMessage {
+  id: string;
+  originalText: string;
+  timestamp: number;
+  isInput: boolean;
+  highlights: { pattern: string; color: string; isRegex: boolean }[];
+  processedContent: string;
+  lines: ProcessedLine[];
+  // Cache for different UI settings
+  processedCache: Map<string, { content: string; lines: ProcessedLine[] }>;
+}
+
+/**
+ * Process ANSI color codes
+ */
+export function processAnsi(text: string, ansiEnabled: boolean): string {
+  if (ansiEnabled) {
+    try {
+      // First process ANSI to HTML without replacing newlines
+      const ansiProcessed = ansiConverter.toHtml(text);
+      
+      // Then replace newlines with <br> tags
+      return ansiProcessed.replace(/\r\n|\r|\n/g, '<br>');
+    } catch (error) {
+      console.error('Error processing ANSI colors:', error);
+      // Fallback to just replacing newlines
+      return text.replace(/\r\n|\r|\n/g, '<br>');
+    }
+  } else {
+    // Strip ANSI codes if color is disabled
+    return text.replace(/\u001b\[\d+(;\d+)*m/g, '')
+              .replace(/\r\n|\r|\n/g, '<br>');
+  }
+}
+
+/**
+ * Split text into individual lines (for better screen reader navigation)
+ */
+export function splitIntoLines(text: string): string[] {
+  // First handle any text that already has <br> tags from ANSI processing
+  if (text.includes('<br>')) {
+    return text.split('<br>').filter(line => line.trim().length > 0);
+  }
+  
+  // Otherwise split by newlines
+  return text.split(/\r\n|\r|\n/).filter(line => line.trim().length > 0);
+}
+
+/**
+ * Apply highlighting to text
+ */
+export function applyHighlights(text: string, highlights: { pattern: string; color: string; isRegex: boolean }[]): string {
+  if (!highlights || highlights.length === 0) return text;
+  
+  let highlightedText = text;
+  
+  highlights.forEach(({ pattern, color, isRegex }) => {
+    if (isRegex) {
+      try {
+        const regex = new RegExp(pattern, 'g');
+        highlightedText = highlightedText.replace(regex, (match) => {
+          return `<span style="background-color: ${color};">${match}</span>`;
+        });
+      } catch (error) {
+        console.error('Invalid regex pattern:', pattern, error);
+      }
+    } else {
+      // Escape special characters in the pattern for use in a regex
+      const safePattern = pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+      const regex = new RegExp(safePattern, 'g');
+      highlightedText = highlightedText.replace(regex, (match) => {
+        return `<span style="background-color: ${color};">${match}</span>`;
+      });
+    }
+  });
+  
+  return highlightedText;
+}
+
+/**
+ * Create a cache key for UI settings that affect text processing
+ */
+export function createCacheKey(ansiEnabled: boolean): string {
+  return `ansi:${ansiEnabled}`;
+}
+
+/**
+ * Process a message completely with caching
+ */
+export function processMessage(
+  message: { 
+    id: string; 
+    text: string; 
+    timestamp: number; 
+    isInput?: boolean; 
+    highlights?: { pattern: string; color: string; isRegex: boolean }[] 
+  },
+  ansiEnabled: boolean
+): ProcessedMessage {
+  const processedMessage: ProcessedMessage = {
+    id: message.id,
+    originalText: message.text,
+    timestamp: message.timestamp,
+    isInput: message.isInput || false,
+    highlights: message.highlights || [],
+    processedContent: '',
+    lines: [],
+    processedCache: new Map()
+  };
+
+  const cacheKey = createCacheKey(ansiEnabled);
+  
+  // Check if we have cached processed content for these settings
+  const cached = processedMessage.processedCache.get(cacheKey);
+  if (cached) {
+    processedMessage.processedContent = cached.content;
+    processedMessage.lines = cached.lines;
+    return processedMessage;
+  }
+
+  // Process the content
+  const ansiProcessed = processAnsi(message.text, ansiEnabled);
+  const highlighted = applyHighlights(ansiProcessed, message.highlights || []);
+  const lines = splitIntoLines(highlighted);
+
+  processedMessage.processedContent = highlighted;
+
+  // Create processed line objects
+  if (lines.length <= 1) {
+    // Single line or no lines
+    processedMessage.lines = [{
+      id: `${message.id}-line-0`,
+      content: highlighted,
+      isSubline: false,
+      parentId: message.id,
+      lineIndex: 0
+    }];
+  } else {
+    // Multiple lines
+    processedMessage.lines = lines.map((line, index) => ({
+      id: `${message.id}-line-${index}`,
+      content: line,
+      isSubline: index > 0,
+      parentId: message.id,
+      lineIndex: index
+    }));
+  }
+
+  // Cache the result
+  processedMessage.processedCache.set(cacheKey, {
+    content: highlighted,
+    lines: processedMessage.lines
+  });
+
+  return processedMessage;
+}