fix: mdx improvements

Author: maxprilutskiy

.changeset/angry-mugs-fold.md

@@ -0,0 +1,5 @@
+---
+"lingo.dev": patch
+---
+
+mdx improvements

packages/cli/src/cli/cmd/i18n.ts

@@ -19,7 +19,7 @@ import chalk from "chalk";
 import { createTwoFilesPatch } from "diff";
 import inquirer from "inquirer";
 import externalEditor from "external-editor";
-import { cacheChunk, deleteCache, getNormalizedCache } from "../utils/cache";
+// import { cacheChunk, deleteCache, getNormalizedCache } from "../utils/cache";
 import updateGitignore from "../utils/update-gitignore";
 import createProcessor from "../processor";
 import { withExponentialBackoff } from "../utils/exp-backoff";
@@ -524,7 +524,7 @@ export default new Command()
                     targetData,
                   },
                   (progress, sourceChunk, processedChunk) => {
-                    cacheChunk(targetLocale, sourceChunk, processedChunk);
+                    // cacheChunk(targetLocale, sourceChunk, processedChunk);
 
                     const progressLog = `[${sourceLocale} -> ${targetLocale}] [${Object.keys(processableData).length} entries] (${progress}%) AI localization in progress...`;
                     if (flags.verbose) {
@@ -615,10 +615,10 @@ export default new Command()
       console.log();
       if (!hasErrors) {
         ora.succeed("Localization completed.");
-        deleteCache();
-        if (flags.verbose) {
-          ora.info("Cache file deleted.");
-        }
+        // deleteCache();
+        // if (flags.verbose) {
+        //   ora.info("Cache file deleted.");
+        // }
         trackEvent(auth.id, "cmd.i18n.success", {
           i18nConfig,
           flags,

packages/cli/src/cli/loaders/index.ts

@@ -31,7 +31,10 @@ import createPhpLoader from "./php";
 import createVueJsonLoader from "./vue-json";
 import createInjectLocaleLoader from "./inject-locale";
 import createLockedKeysLoader from "./locked-keys";
-import createMdxLoader from "./mdx2";
+import createMdxFrontmatterSplitLoader from "./mdx2/frontmatter-split";
+import createMdxCodePlaceholderLoader from "./mdx2/code-placeholder";
+import createLocalizableMdxDocumentLoader from "./mdx2/localizable-document";
+import createMdxSectionsSplit2Loader from "./mdx2/sections-split-2";
 
 type BucketLoaderOptions = {
   isCacheRestore: boolean;
@@ -110,8 +113,14 @@ export default function createBucketLoader(
     case "mdx":
       return composeLoaders(
         createTextFileLoader(bucketPathPattern),
-        createPrettierLoader({ parser: "mdx", bucketPathPattern }),
-        createMdxLoader(),
+        createMdxCodePlaceholderLoader(),
+        createPrettierLoader({
+          parser: "mdx",
+          bucketPathPattern,
+        }),
+        createMdxFrontmatterSplitLoader(),
+        createMdxSectionsSplit2Loader(),
+        createLocalizableMdxDocumentLoader(),
         createFlatLoader(),
         createSyncLoader(),
         createUnlocalizableLoader(

packages/cli/src/cli/loaders/mdx2/_utils.ts

@@ -1,9 +1,4 @@
 import { Root, RootContent } from "mdast";
-import { VFile } from "vfile";
-import { visit } from "unist-util-visit";
-import { fromMarkdown } from "mdast-util-from-markdown";
-import { gfm } from "micromark-extension-gfm";
-import { gfmFromMarkdown } from "mdast-util-gfm";
 
 export function traverseMdast(
   ast: Root | RootContent,

packages/cli/src/cli/loaders/mdx2/code-placeholder.ts

@@ -1,15 +1,6 @@
-import {
-  Root,
-  Paragraph,
-  RootContent,
-  RootContentMap,
-  InlineCode,
-  Code,
-} from "mdast";
 import { ILoader } from "../_types";
 import { createLoader } from "../_utils";
 import { PlaceholderedMdx, RawMdx } from "./_types";
-import { traverseMdast } from "./_utils";
 import { md5 } from "../../utils/md5";
 import { unified } from "unified";
 import remarkParse from "remark-parse";
@@ -35,63 +26,19 @@ function extractCodePlaceholders(content: string): {
   content: string;
   codePlaceholders: Record<string, string>;
 } {
-  let shouldSearchForPlaceholders = true;
   let finalContent = content;
   const codePlaceholders: Record<string, string> = {};
 
-  while (shouldSearchForPlaceholders) {
-    const ast = parseMdast(finalContent);
+  const codeBlockRegex = /^```.*\n([\s\S]*?)^```$/gm;
+  const codeBlockMatches = finalContent.matchAll(codeBlockRegex);
 
-    traverseMdast(ast, (_node) => {
-      shouldSearchForPlaceholders = false;
+  for (const match of codeBlockMatches) {
+    const codeBlock = match[0];
+    const codeBlockHash = md5(codeBlock);
+    const placeholderId = `---CODE_PLACEHOLDER_${codeBlockHash}---`;
 
-      if (_node.type === "code") {
-        const node = _node as Code;
-
-        const nodeContentStartLine = node.position?.start.line;
-        const nodeContentEndLine = node.position?.end.line;
-
-        if (!nodeContentStartLine || !nodeContentEndLine) {
-          return;
-        }
-
-        const nodeContentPreStartLine = nodeContentStartLine - 1;
-        const nodeContentPostEndLine = nodeContentEndLine + 1;
-
-        const nodeContent = finalContent
-          .split("\n")
-          .slice(nodeContentPreStartLine, nodeContentPostEndLine)
-          .join("\n");
-
-        const nodeContentHash = md5(nodeContent);
-        const placeholderId = `__PLACEHOLDER_${nodeContentHash}__`;
-
-        codePlaceholders[placeholderId] = nodeContent;
-        finalContent = finalContent.replace(nodeContent, placeholderId);
-        shouldSearchForPlaceholders = true;
-      } else if (_node.type === "inlineCode") {
-        const node = _node as InlineCode;
-
-        const nodeContentStartIndex = node.position?.start.offset;
-        const nodeContentEndIndex = node.position?.end.offset;
-
-        if (!nodeContentStartIndex || !nodeContentEndIndex) {
-          return;
-        }
-
-        const nodeContent = finalContent.slice(
-          nodeContentStartIndex,
-          nodeContentEndIndex,
-        );
-
-        const nodeContentHash = md5(nodeContent);
-        const placeholderId = `__PLACEHOLDER_${nodeContentHash}__`;
-
-        codePlaceholders[placeholderId] = nodeContent;
-        finalContent = finalContent.replace(nodeContent, placeholderId);
-        shouldSearchForPlaceholders = true;
-      }
-    });
+    codePlaceholders[placeholderId] = codeBlock;
+    finalContent = finalContent.replace(codeBlock, placeholderId);
   }
 
   return {
@@ -101,49 +48,25 @@ function extractCodePlaceholders(content: string): {
 }
 
 export default function createMdxCodePlaceholderLoader(): ILoader<
-  RawMdx,
-  PlaceholderedMdx
+  string,
+  string
 > {
   return createLoader({
     async pull(locale, input) {
-      const { frontmatter = {}, content = "" } = input || {
-        frontmatter: {},
-        content: "",
-      };
-
-      const { content: resultContent, codePlaceholders } =
-        extractCodePlaceholders(content);
-
-      return {
-        frontmatter,
-        content: resultContent,
-        codePlaceholders,
-      };
+      const response = extractCodePlaceholders(input);
+      return response.content;
     },
 
     async push(locale, data, originalInput) {
-      // Re-create the placeholders map from the original (unmodified) input so we
-      // can rely on a stable mapping even if `data.codePlaceholders` was lost or
-      // tampered with by other loaders in the chain.
-      const { codePlaceholders } = extractCodePlaceholders(
-        originalInput?.content ?? "",
-      );
+      const response = extractCodePlaceholders(originalInput ?? "");
 
-      let finalContent = data.content;
-      // Restore code from placeholders
-      for (const [placeholder, original] of Object.entries(codePlaceholders)) {
-        // Use String.replaceAll with a replacer function to restore the original code.
-        // A replacer function is used to avoid any special replacement pattern
-        // interpretation (e.g. "$&", "$$", etc.) that could occur if `original`
-        // contains `$` characters.
-        finalContent = finalContent.replaceAll(placeholder, () => original);
+      let result = data;
+      for (const [placeholder, original] of Object.entries(
+        response.codePlaceholders,
+      )) {
+        result = result.replaceAll(placeholder, original);
       }
 
-      const result: RawMdx = {
-        frontmatter: data.frontmatter,
-        content: finalContent,
-      };
-
       return result;
     },
   });

packages/cli/src/cli/loaders/mdx2/index.ts

@@ -0,0 +1,39 @@
+import { ILoader } from "../_types";
+import { createLoader } from "../_utils";
+import { PlaceholderedMdx, SectionedMdx } from "./_types";
+import _ from "lodash";
+
+export default function createMdxSectionsSplit2Loader(): ILoader<
+  PlaceholderedMdx,
+  SectionedMdx
+> {
+  return createLoader({
+    async pull(locale, input) {
+      const sections = _.chain(input.content)
+        .split("\n\n")
+        .filter(Boolean)
+        .map((section, index) => [index, section])
+        .fromPairs()
+        .value();
+
+      const result: SectionedMdx = {
+        frontmatter: input.frontmatter,
+        sections,
+      };
+
+      return result;
+    },
+
+    async push(locale, data, originalInput, _originalLocale, pullInput) {
+      const content = _.chain(data.sections).values().join("\n\n").value();
+
+      const result: PlaceholderedMdx = {
+        frontmatter: data.frontmatter,
+        codePlaceholders: pullInput?.codePlaceholders || {},
+        content,
+      };
+
+      return result;
+    },
+  });
+}

packages/cli/src/cli/loaders/mdx2/sections-split-2.ts

@@ -6,6 +6,7 @@ import { createLoader } from "./_utils";
 export type PrettierLoaderOptions = {
   parser: Options["parser"];
   bucketPathPattern: string;
+  stage?: "pull" | "push" | "both";
   alwaysFormat?: boolean;
 };
 
@@ -14,6 +15,10 @@ export default function createPrettierLoader(
 ): ILoader<string, string> {
   return createLoader({
     async pull(locale, data) {
+      if (!["pull", "both"].includes(options.stage!)) {
+        return data;
+      }
+
       const draftPath = options.bucketPathPattern.replaceAll(
         "[locale]",
         locale,
@@ -23,6 +28,10 @@ export default function createPrettierLoader(
       return await formatDataWithPrettier(data, finalPath, options);
     },
     async push(locale, data) {
+      if (!["push", "both"].includes(options.stage!)) {
+        return data;
+      }
+
       const draftPath = options.bucketPathPattern.replaceAll(
         "[locale]",
         locale,