Skip to content

Commit c5ccf81

Browse files
devin-ai-integration[bot]maxprilutskiy
devin-ai-integration[bot]
and
maxprilutskiy
authored
feat: add support for locked patterns in mdx loader (#700)
* feat: add support for locked patterns in mdx loader Co-Authored-By: Max Prilutskiy <maks.prilutskiy@gmail.com> * test: update locked patterns tests to use toBe instead of toMatch/toContain Co-Authored-By: Max Prilutskiy <maks.prilutskiy@gmail.com> * fix(mdx): fix locked patterns tests and improve regex patterns Co-Authored-By: Max Prilutskiy <maks.prilutskiy@gmail.com> * fix: make locked patterns configurable via i18n.json Co-Authored-By: Max Prilutskiy <maks.prilutskiy@gmail.com> * refactor: remove default patterns fallback in locked patterns loader Co-Authored-By: Max Prilutskiy <maks.prilutskiy@gmail.com> * refactor: update locked patterns loader to return string and use pullInput Co-Authored-By: Max Prilutskiy <maks.prilutskiy@gmail.com> * refactor: update locked patterns loader to properly use pullInput parameter Co-Authored-By: Max Prilutskiy <maks.prilutskiy@gmail.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Max Prilutskiy <maks.prilutskiy@gmail.com> Co-authored-by: Max Prilutskiy <5614659+maxprilutskiy@users.noreply.github.com>
1 parent 0f3407f commit c5ccf81

File tree

8 files changed

+405
-2
lines changed

8 files changed

+405
-2
lines changed

‎.changeset/blue-pens-swim.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
---
2+
"@lingo.dev/_spec": minor
3+
"lingo.dev": minor
4+
---
5+
6+
Add support for locked patterns in MDX loader
7+
8+
This change adds support for preserving specific patterns in MDX files during translation, including:
9+
10+
- !params syntax for parameter documentation
11+
- !! parameter_name headings
12+
- !type declarations
13+
- !required flags
14+
- !values lists
15+
16+
The implementation adds a new config version 1.7 with a "lockedPatterns" field that accepts an array of regex patterns to be preserved during translation.

‎i18n.json‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version": 1.2,
2+
"version": 1.7,
33
"locale": {
44
"source": "en",
55
"targets": [
@@ -23,6 +23,16 @@
2323
"include": [
2424
"readme/[locale].md"
2525
]
26+
},
27+
"mdx": {
28+
"include": [],
29+
"lockedPatterns": [
30+
"!params",
31+
"!! [\\w_]+",
32+
"!type [\\w<>\\[\\]\"',]+",
33+
"!required",
34+
"!values [\\s\\S]*?(?=\\n\\n|$)"
35+
]
2636
}
2737
},
2838
"$schema": "https://lingo.dev/schema/i18n.json"

‎packages/cli/src/cli/cmd/i18n.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,7 @@ export default new Command()
175175
injectLocale: bucket.injectLocale,
176176
},
177177
bucket.lockedKeys,
178+
bucket.lockedPatterns,
178179
);
179180
bucketLoader.setDefaultLocale(sourceLocale);
180181
await bucketLoader.init();
@@ -454,6 +455,7 @@ export default new Command()
454455
injectLocale: bucket.injectLocale,
455456
},
456457
bucket.lockedKeys,
458+
bucket.lockedPatterns,
457459
);
458460
bucketLoader.setDefaultLocale(sourceLocale);
459461
await bucketLoader.init();

‎packages/cli/src/cli/loaders/index.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import createMdxFrontmatterSplitLoader from "./mdx2/frontmatter-split";
3636
import createMdxCodePlaceholderLoader from "./mdx2/code-placeholder";
3737
import createLocalizableMdxDocumentLoader from "./mdx2/localizable-document";
3838
import createMdxSectionsSplit2Loader from "./mdx2/sections-split-2";
39+
import createMdxLockedPatternsLoader from "./mdx2/locked-patterns";
3940

4041
type BucketLoaderOptions = {
4142
isCacheRestore: boolean;
@@ -49,6 +50,7 @@ export default function createBucketLoader(
4950
bucketPathPattern: string,
5051
options: BucketLoaderOptions,
5152
lockedKeys?: string[],
53+
lockedPatterns?: string[],
5254
): ILoader<void, Record<string, string>> {
5355
switch (bucketType) {
5456
default:
@@ -119,6 +121,7 @@ export default function createBucketLoader(
119121
bucketPathPattern,
120122
}),
121123
createMdxCodePlaceholderLoader(),
124+
createMdxLockedPatternsLoader(lockedPatterns),
122125
createMdxFrontmatterSplitLoader(),
123126
createMdxSectionsSplit2Loader(),
124127
createLocalizableMdxDocumentLoader(),

‎packages/cli/src/cli/loaders/mdx2/locked-patterns.spec.ts‎

Lines changed: 266 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,266 @@
1+
import { describe, it, expect } from "vitest";
2+
import createMdxLockedPatternsLoader from "./locked-patterns";
3+
import dedent from "dedent";
4+
5+
describe("MDX Locked Patterns Loader", () => {
6+
describe("Basic functionality", () => {
7+
it("should do nothing when no patterns are provided", async () => {
8+
const loader = createMdxLockedPatternsLoader();
9+
loader.setDefaultLocale("en");
10+
11+
const md = dedent`
12+
# Title
13+
14+
Some content.
15+
16+
!params
17+
18+
!! parameter_name
19+
20+
!type string
21+
`;
22+
23+
const result = await loader.pull("en", md);
24+
25+
const placeholderRegex = /---LOCKED-PATTERN-[0-9a-f]+---/g;
26+
const placeholders = result.match(placeholderRegex) || [];
27+
expect(placeholders.length).toBe(0); // No patterns should be replaced
28+
29+
expect(result).toBe(md);
30+
31+
const pushed = await loader.push("es", result);
32+
expect(pushed).toBe(md);
33+
});
34+
35+
it("should preserve content matching patterns", async () => {
36+
const loader = createMdxLockedPatternsLoader([
37+
"!params",
38+
"!! [\\w_]+",
39+
"!type [\\w<>\\[\\]\"',]+"
40+
]);
41+
loader.setDefaultLocale("en");
42+
43+
const md = dedent`
44+
# Title
45+
46+
Some content.
47+
48+
!params
49+
50+
!! parameter_name
51+
52+
!type string
53+
`;
54+
55+
const result = await loader.pull("en", md);
56+
57+
const placeholderRegex = /---LOCKED-PATTERN-[0-9a-f]+---/g;
58+
const placeholders = result.match(placeholderRegex) || [];
59+
expect(placeholders.length).toBe(3); // Three patterns should be replaced
60+
61+
const sanitizedContent = result
62+
.replace(placeholderRegex, "---PLACEHOLDER---");
63+
64+
const expectedSanitized = dedent`
65+
# Title
66+
67+
Some content.
68+
69+
---PLACEHOLDER---
70+
71+
---PLACEHOLDER---
72+
73+
---PLACEHOLDER---
74+
`;
75+
76+
expect(sanitizedContent).toBe(expectedSanitized);
77+
78+
const translated = result
79+
.replace("# Title", "# Título")
80+
.replace("Some content.", "Algún contenido.");
81+
82+
const pushed = await loader.push("es", translated);
83+
84+
const expectedPushed = dedent`
85+
# Título
86+
87+
Algún contenido.
88+
89+
!params
90+
91+
!! parameter_name
92+
93+
!type string
94+
`;
95+
96+
expect(pushed).toBe(expectedPushed);
97+
});
98+
});
99+
100+
describe("Real-world patterns", () => {
101+
it("should handle !hover syntax in code blocks", async () => {
102+
const loader = createMdxLockedPatternsLoader([
103+
"// !hover[\\s\\S]*?(?=\\n|$)",
104+
"// !hover\\([\\d:]+\\)[\\s\\S]*?(?=\\n|$)"
105+
]);
106+
loader.setDefaultLocale("en");
107+
108+
const md = dedent`
109+
\`\`\`js
110+
const x = 1;
111+
const pubkey = "vines1vzrYbzLMRdu58ou5XTby4qAqVRLmqo36NKPTg";
112+
\`\`\`
113+
`;
114+
115+
const result = await loader.pull("en", md);
116+
117+
const placeholderRegex = /---LOCKED-PATTERN-[0-9a-f]+---/g;
118+
const placeholders = result.match(placeholderRegex) || [];
119+
expect(placeholders.length).toBe(0); // No patterns should be replaced
120+
121+
const pushed = await loader.push("es", result);
122+
123+
expect(pushed).toBe(md);
124+
});
125+
126+
it("should handle !! parameter headings", async () => {
127+
const loader = createMdxLockedPatternsLoader([
128+
"!! [\\w_]+"
129+
]);
130+
loader.setDefaultLocale("en");
131+
132+
const md = dedent`
133+
# Parameters
134+
135+
!! pubkey
136+
137+
The public key of the account to query.
138+
139+
!! encoding
140+
141+
Encoding format for the returned Account data.
142+
`;
143+
144+
const result = await loader.pull("en", md);
145+
146+
const placeholderRegex = /---LOCKED-PATTERN-[0-9a-f]+---/g;
147+
const placeholders = result.match(placeholderRegex) || [];
148+
expect(placeholders.length).toBe(2); // Two patterns should be replaced
149+
150+
const sanitizedContent = result
151+
.replace(placeholderRegex, "---PLACEHOLDER---");
152+
153+
const expectedSanitized = dedent`
154+
# Parameters
155+
156+
---PLACEHOLDER---
157+
158+
The public key of the account to query.
159+
160+
---PLACEHOLDER---
161+
162+
Encoding format for the returned Account data.
163+
`;
164+
165+
expect(sanitizedContent).toBe(expectedSanitized);
166+
167+
const translated = result
168+
.replace("# Parameters", "# Parámetros")
169+
.replace("The public key of the account to query.", "La clave pública de la cuenta a consultar.")
170+
.replace("Encoding format for the returned Account data.", "Formato de codificación para los datos de la cuenta devueltos.");
171+
172+
const pushed = await loader.push("es", translated);
173+
174+
const expectedPushed = dedent`
175+
# Parámetros
176+
177+
!! pubkey
178+
179+
La clave pública de la cuenta a consultar.
180+
181+
!! encoding
182+
183+
Formato de codificación para los datos de la cuenta devueltos.
184+
`;
185+
186+
expect(pushed).toBe(expectedPushed);
187+
});
188+
189+
it("should handle !type, !required, and !values declarations", async () => {
190+
const loader = createMdxLockedPatternsLoader([
191+
"!! [\\w_]+",
192+
"!type [\\w<>\\[\\]\"',]+",
193+
"!required",
194+
"!values [\\s\\S]*?(?=\\n\\n|$)"
195+
]);
196+
loader.setDefaultLocale("en");
197+
198+
const md = dedent`
199+
!! pubkey
200+
201+
!type string
202+
!required
203+
204+
The public key of the account to query.
205+
206+
!! encoding
207+
208+
!type string
209+
!values "base58" (default), "base64", "jsonParsed"
210+
211+
Encoding format for the returned Account data.
212+
`;
213+
214+
const result = await loader.pull("en", md);
215+
216+
const placeholderRegex = /---LOCKED-PATTERN-[0-9a-f]+---/g;
217+
const placeholders = result.match(placeholderRegex) || [];
218+
expect(placeholders.length).toBe(6); // Six patterns should be replaced
219+
220+
const sanitizedContent = result
221+
.replace(placeholderRegex, "---PLACEHOLDER---");
222+
223+
const expectedSanitized = dedent`
224+
---PLACEHOLDER---
225+
226+
---PLACEHOLDER---
227+
---PLACEHOLDER---
228+
229+
The public key of the account to query.
230+
231+
---PLACEHOLDER---
232+
233+
---PLACEHOLDER---
234+
---PLACEHOLDER---
235+
236+
Encoding format for the returned Account data.
237+
`;
238+
239+
expect(sanitizedContent).toBe(expectedSanitized);
240+
241+
const translated = result
242+
.replace("The public key of the account to query.", "La clave pública de la cuenta a consultar.")
243+
.replace("Encoding format for the returned Account data.", "Formato de codificación para los datos de la cuenta devueltos.");
244+
245+
const pushed = await loader.push("es", translated);
246+
247+
const expectedPushed = dedent`
248+
!! pubkey
249+
250+
!type string
251+
!required
252+
253+
La clave pública de la cuenta a consultar.
254+
255+
!! encoding
256+
257+
!type string
258+
!values "base58" (default), "base64", "jsonParsed"
259+
260+
Formato de codificación para los datos de la cuenta devueltos.
261+
`;
262+
263+
expect(pushed).toBe(expectedPushed);
264+
});
265+
});
266+
});

0 commit comments

Comments
 (0)