[APIView] Sanitize text tokens of newlines on ingestion (#15081)

* [APIView] Sanitize text tokens of newlines on ingestion

* respond to copilot

* Respond to feedback

Author: smw-ms

src/dotnet/APIView/APIViewUnitTests/CodeFileManagerTests.cs

@@ -415,4 +415,104 @@ public async Task ComputeAPIContentHashAsync_ProducesDifferentHash_WhenApiSurfac
     }
 
     #endregion
+
+    #region Token Sanitization Tests
+
+    [Fact]
+    public async Task CreateReviewCodeFileModel_SanitizesNewlinesInTreeTokenValues()
+    {
+        var codeFile = new CodeFile
+        {
+            Language = "Java",
+            PackageName = "com.azure.storage",
+            ReviewLines =
+            [
+                new ReviewLine
+                {
+                    LineId = "doc-1",
+                    Tokens =
+                    [
+                        new ReviewToken
+                        {
+                            Value = "\n    This package contains clients.\n    For details see README.md\n  ",
+                            Kind = TokenKind.Text,
+                            IsDocumentation = true
+                        }
+                    ]
+                }
+            ]
+        };
+
+        using var memoryStream = new MemoryStream();
+
+        await _codeFileManager.CreateReviewCodeFileModel("api-rev-1", memoryStream, codeFile);
+
+        string value = codeFile.ReviewLines[0].Tokens[0].Value;
+        Assert.Equal("     This package contains clients.     For details see README.md   ", value);
+        Assert.DoesNotContain('\n', value);
+        Assert.DoesNotContain('\r', value);
+    }
+
+    [Fact]
+    public async Task CreateReviewCodeFileModel_SanitizesOnlyNestedTextTokens()
+    {
+        var codeFile = new CodeFile
+        {
+            Language = "C#",
+            ReviewLines =
+            [
+                new ReviewLine
+                {
+                    LineId = "parent",
+                    Tokens = [new ReviewToken("Parent\ntoken", TokenKind.Keyword)],
+                    Children =
+                    [
+                        new ReviewLine
+                        {
+                            LineId = "child",
+                            Tokens = [new ReviewToken("Child\ntoken\nvalue", TokenKind.Text)]
+                        }
+                    ]
+                }
+            ]
+        };
+
+        using var memoryStream = new MemoryStream();
+
+        await _codeFileManager.CreateReviewCodeFileModel("api-rev-3", memoryStream, codeFile);
+
+        Assert.Equal("Parent\ntoken", codeFile.ReviewLines[0].Tokens[0].Value);
+        Assert.Equal("Child token value", codeFile.ReviewLines[0].Children[0].Tokens[0].Value);
+    }
+
+    [Fact]
+    public async Task CreateReviewCodeFileModel_DoesNotModifyTreeTokenWithoutNewlines()
+    {
+        var codeFile = new CodeFile
+        {
+            Language = "Java",
+            ReviewLines =
+            [
+                new ReviewLine
+                {
+                    LineId = "tree-1",
+                    Tokens =
+                    [
+                        new ReviewToken("    ", TokenKind.Text),
+                        new ReviewToken("NoNewlines", TokenKind.Text)
+                    ]
+                }
+            ]
+        };
+
+        using var memoryStream = new MemoryStream();
+
+        await _codeFileManager.CreateReviewCodeFileModel("api-rev-3a", memoryStream, codeFile);
+
+        Assert.Equal("    ", codeFile.ReviewLines[0].Tokens[0].Value);
+        Assert.Equal("NoNewlines", codeFile.ReviewLines[0].Tokens[1].Value);
+    }
+
+    #endregion
+
 }

src/dotnet/APIView/APIViewWeb/Managers/CodeFileManager.cs

@@ -217,6 +217,9 @@ public async Task<APICodeFileModel> CreateReviewCodeFileModel(string apiRevision
                 memoryStream.Position = 0;
                 await _originalsRepository.UploadOriginalAsync(reviewCodeFileModel.FileId, memoryStream);
             }
+            
+            SanitizeTokenValues(codeFile);
+
             await _codeFileRepository.UpsertCodeFileAsync(apiRevisionId, reviewCodeFileModel.FileId, codeFile);
             reviewCodeFileModel.ContentHash = await ComputeAPIContentHashAsync(codeFile);
             return reviewCodeFileModel;
@@ -316,5 +319,53 @@ private static void InitializeFromCodeFile(APICodeFileModel file, CodeFile codeF
             file.CrossLanguagePackageId = codeFile.CrossLanguageMetadata != null ? codeFile.CrossLanguageMetadata.CrossLanguagePackageId : codeFile.CrossLanguagePackageId;
             file.ParserStyle = (codeFile.ReviewLines.Count > 0) ? ParserStyle.Tree : ParserStyle.Flat;
         }
+
+        /// <summary>
+        /// Sanitizes tree-style token values in the CodeFile by removing embedded newlines.
+        /// Legacy flat tokens are intentionally left unchanged.
+        /// </summary>
+        private static void SanitizeTokenValues(CodeFile codeFile)
+        {
+            // Tree-style (ReviewToken is a class, so mutate in place)
+            if (codeFile.ReviewLines != null && codeFile.ReviewLines.Count > 0)
+            {
+                SanitizeReviewLines(codeFile.ReviewLines);
+            }
+        }
+
+        /// <summary>
+        /// Recursively sanitizes token values in ReviewLines and their children.
+        /// ReviewToken is a reference type, so mutations are applied in place.
+        /// </summary>
+        private static void SanitizeReviewLines(List<ReviewLine> lines)
+        {
+            foreach (var line in lines)
+            {
+                foreach (var token in line.Tokens)
+                {
+                    if (token.Kind == TokenKind.Text && !string.IsNullOrEmpty(token.Value))
+                        token.Value = NormalizeTokenValue(token.Value);
+                }
+                if (line.Children != null && line.Children.Count > 0)
+                {
+                    SanitizeReviewLines(line.Children);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Normalizes a token value by replacing all newline characters (both \r\n and \n\r)
+        /// with single spaces, then trimming leading/trailing whitespace.
+        /// </summary>
+        private static string NormalizeTokenValue(string value)
+        {
+            if (string.IsNullOrEmpty(value))
+                return value;
+
+            return value
+                .Replace("\r\n", " ")
+                .Replace('\r', ' ')
+                .Replace('\n', ' ');
+        }
     }
 }