Skip to content

Commit 04c3679

Browse files
cherkanovartvrcprl
cherkanovart
and
vrcprl
authored
feat(cli): add csv-per-locale bucket type (#1742)
* feat(cli): add CSV per locale loader and associated i18n configurations * feat(cli): enhance i18n support with per-locale CSV configuration and new demo files * chore(cli): add changeset for csv-per-locale loader update * fix(cli): normalize CSV patterns in bucket loader and update i18n configurations * refactor(cli): update CSV files for English and Spanish locales with new structure and content --------- Co-authored-by: Veronica Prilutskaya <veronica@lingo.dev>
1 parent 348b2de commit 04c3679

11 files changed

Lines changed: 370 additions & 0 deletions

File tree

‎.changeset/perky-beers-travel.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
"@lingo.dev/_spec": minor
3+
"lingo.dev": minor
4+
---
5+
6+
Add csv-per-locale bucket and improve ignoredKeys support for CSV

‎packages/cli/demo/csv-per-locale/en/example.csv‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
id,name,description,created,enabled,sort
2+
1,Welcome,Welcome to our application,2024-01-01,true,1
3+
2,Save,Save your changes,2024-01-01,true,2
4+
3,Error,An error occurred,2024-01-01,true,3
5+
4,Success,Operation completed successfully,2024-01-01,true,4
6+
5,Loading,Please wait while we load your data,2024-01-01,true,5

‎packages/cli/demo/csv-per-locale/es/example.csv‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
id,name,description,created,enabled,sort
2+
1,Bienvenida,Bienvenido a nuestra aplicación,2024-01-01,true,1
3+
2,Guardar,Guarda tus cambios,2024-01-01,true,2
4+
3,Error,Ha ocurrido un error,2024-01-01,true,3
5+
4,Éxito,Operación completada con éxito,2024-01-01,true,4
6+
5,Cargando,Por favor espera mientras cargamos tus datos,2024-01-01,true,5

‎packages/cli/demo/csv-per-locale/i18n.json‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"version": "1.10",
3+
"locale": {
4+
"source": "en",
5+
"targets": ["es"]
6+
},
7+
"buckets": {
8+
"csv-per-locale": {
9+
"include": ["./[locale]/example.csv"],
10+
"lockedKeys": ["locked_key_1"],
11+
"ignoredKeys": ["ignored_key_1"]
12+
}
13+
},
14+
"$schema": "https://lingo.dev/schema/i18n.json"
15+
}

‎packages/cli/demo/csv-per-locale/i18n.lock‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
version: 1
2+
checksums:
3+
e8b273672f895de0944f0a2317670d7c:
4+
0/NAME: 82fd7edcad911fd528a6e1d65905e468
5+
0/PRODUCT: 97cddc0ea1a642dbbc77e2c58ef96c54
6+
0/BONUS: 3c196cabd552b6b6941f8813562cdec6
7+
1/NAME: 3e7241ad69e00f71f6447b69006fdd6b
8+
1/PRODUCT: 52dceb6e133876eedefc91758f4dafa8
9+
1/BONUS: a61657f3137c31deb9498810287d7ed4
10+
2/NAME: 229dc77b9845261d704d992c7b00153a
11+
2/PRODUCT: 45b9756cad31c05eba897f19a4550ecb
12+
2/BONUS: 02ac38cc1479911157054402e82959c1
13+
3/NAME: 40f55d4ddad53f1c31c7244813bc68e7
14+
3/PRODUCT: 93017112c1dc2f074d3be1abb02d2507
15+
3/BONUS: 604e6eca8e6c781586fab7cf0b97f0b7
16+
4/NAME: ba44c67983adfdd5bf03ec5168f3abc9
17+
4/PRODUCT: 898e5908c9726f8056673258dbe9b1af
18+
4/BONUS: 719f3dbab3cb569a93518d4c2ff1b633
19+
5/NAME: 61d99435646f27866c505e7d1f40d171
20+
5/PRODUCT: e9c4963b1da635d2365e0111a7a7fc2f
21+
5/BONUS: 35db06a282738c6c6a9417094d39c80e
22+
6/NAME: 032412ef2ec37e8be14137292049e970
23+
6/PRODUCT: 221ea6f7cf3778ae8b9f079588d7fb7a
24+
6/BONUS: b557d0a6619c27e558b8581ce7d3108a
25+
7/NAME: 6a554c4b466acc8b76d043452e3a710f
26+
7/PRODUCT: e6b44d2244ead5e4ae5a6a3755d103f8
27+
7/BONUS: 0a91164a59598e2650e4e48eaa6bd4bf
28+
8/NAME: 275a03d2e25a65f126991ada1daa870d
29+
8/PRODUCT: 97cddc0ea1a642dbbc77e2c58ef96c54
30+
8/BONUS: 3c196cabd552b6b6941f8813562cdec6
31+
9/NAME: 221b270fe5a60e5a160d5502f9b0c139
32+
9/PRODUCT: 52dceb6e133876eedefc91758f4dafa8
33+
9/BONUS: a61657f3137c31deb9498810287d7ed4
34+
10/NAME: 7517a402f3581ea2c04a992b8468c008
35+
10/PRODUCT: 45b9756cad31c05eba897f19a4550ecb
36+
10/BONUS: 02ac38cc1479911157054402e82959c1
37+
11/NAME: a649692aa9ba9468f98ed718bd4d0729
38+
11/PRODUCT: 93017112c1dc2f074d3be1abb02d2507
39+
11/BONUS: 604e6eca8e6c781586fab7cf0b97f0b7
40+
12/NAME: 10e10057922963a0ef53b526cb2baf90
41+
12/PRODUCT: 898e5908c9726f8056673258dbe9b1af
42+
12/BONUS: 719f3dbab3cb569a93518d4c2ff1b633
43+
13/NAME: 2b0365cf946aeb2678cb2a10f7b662ae
44+
13/PRODUCT: e9c4963b1da635d2365e0111a7a7fc2f
45+
13/BONUS: 35db06a282738c6c6a9417094d39c80e
46+
14/NAME: bbe0e79469d425ce26a44bd8c1f9783d
47+
14/PRODUCT: 221ea6f7cf3778ae8b9f079588d7fb7a
48+
14/BONUS: b557d0a6619c27e558b8581ce7d3108a
49+
15/NAME: e08118de3644266e79c63d1fe03bdd34
50+
15/PRODUCT: e6b44d2244ead5e4ae5a6a3755d103f8
51+
15/BONUS: 0a91164a59598e2650e4e48eaa6bd4bf

‎packages/cli/i18n.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,11 @@
1717
"csv": {
1818
"include": ["demo/csv/example.csv"]
1919
},
20+
"csv-per-locale": {
21+
"include": ["demo/csv-per-locale/[locale]/example.csv"],
22+
"lockedKeys": ["locked_key_1"],
23+
"ignoredKeys": ["ignored_key_1"]
24+
},
2025
"ejs": {
2126
"include": ["demo/ejs/[locale]/*.ejs"]
2227
},

‎packages/cli/i18n.lock‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -815,3 +815,68 @@ checksums:
815815
body/1/3/2: 0429f12258fabbde3abaca3dd9986178
816816
body/2/0: d32e57e4a5a65f3bee8b63dcb2bfa8e7
817817
body/2/1: 7e10a8ab9cc4e6d603b3cdc48849688f
818+
0b4cc73cf7debceac50ede9a7c731c9a:
819+
0/NAME: 82fd7edcad911fd528a6e1d65905e468
820+
0/PRODUCT: 97cddc0ea1a642dbbc77e2c58ef96c54
821+
0/BONUS: 3c196cabd552b6b6941f8813562cdec6
822+
0/BONUS__2: 34cd669195d2ca8c293aa2191417a1d4
823+
1/NAME: 3e7241ad69e00f71f6447b69006fdd6b
824+
1/PRODUCT: 52dceb6e133876eedefc91758f4dafa8
825+
1/BONUS: a61657f3137c31deb9498810287d7ed4
826+
1/BONUS__2: 34cd669195d2ca8c293aa2191417a1d4
827+
2/NAME: 229dc77b9845261d704d992c7b00153a
828+
2/PRODUCT: 45b9756cad31c05eba897f19a4550ecb
829+
2/BONUS: 02ac38cc1479911157054402e82959c1
830+
2/BONUS__2: 34cd669195d2ca8c293aa2191417a1d4
831+
3/NAME: 40f55d4ddad53f1c31c7244813bc68e7
832+
3/PRODUCT: 93017112c1dc2f074d3be1abb02d2507
833+
3/BONUS: 604e6eca8e6c781586fab7cf0b97f0b7
834+
3/BONUS__2: 34cd669195d2ca8c293aa2191417a1d4
835+
4/NAME: ba44c67983adfdd5bf03ec5168f3abc9
836+
4/PRODUCT: 898e5908c9726f8056673258dbe9b1af
837+
4/BONUS: 719f3dbab3cb569a93518d4c2ff1b633
838+
4/BONUS__2: 9762a40457f1695e8679def406f76d37
839+
5/NAME: 61d99435646f27866c505e7d1f40d171
840+
5/PRODUCT: e9c4963b1da635d2365e0111a7a7fc2f
841+
5/BONUS: 35db06a282738c6c6a9417094d39c80e
842+
5/BONUS__2: 34cd669195d2ca8c293aa2191417a1d4
843+
6/NAME: 032412ef2ec37e8be14137292049e970
844+
6/PRODUCT: 221ea6f7cf3778ae8b9f079588d7fb7a
845+
6/BONUS: b557d0a6619c27e558b8581ce7d3108a
846+
6/BONUS__2: 34cd669195d2ca8c293aa2191417a1d4
847+
7/NAME: 6a554c4b466acc8b76d043452e3a710f
848+
7/PRODUCT: e6b44d2244ead5e4ae5a6a3755d103f8
849+
7/BONUS: 0a91164a59598e2650e4e48eaa6bd4bf
850+
7/BONUS__2: 34cd669195d2ca8c293aa2191417a1d4
851+
8/NAME: 275a03d2e25a65f126991ada1daa870d
852+
8/PRODUCT: 97cddc0ea1a642dbbc77e2c58ef96c54
853+
8/BONUS: 3c196cabd552b6b6941f8813562cdec6
854+
8/BONUS__2: 34cd669195d2ca8c293aa2191417a1d4
855+
9/NAME: 221b270fe5a60e5a160d5502f9b0c139
856+
9/PRODUCT: 52dceb6e133876eedefc91758f4dafa8
857+
9/BONUS: a61657f3137c31deb9498810287d7ed4
858+
9/BONUS__2: 34cd669195d2ca8c293aa2191417a1d4
859+
10/NAME: 7517a402f3581ea2c04a992b8468c008
860+
10/PRODUCT: 45b9756cad31c05eba897f19a4550ecb
861+
10/BONUS: 02ac38cc1479911157054402e82959c1
862+
10/BONUS__2: 34cd669195d2ca8c293aa2191417a1d4
863+
11/NAME: a649692aa9ba9468f98ed718bd4d0729
864+
11/PRODUCT: 93017112c1dc2f074d3be1abb02d2507
865+
11/BONUS: 604e6eca8e6c781586fab7cf0b97f0b7
866+
11/BONUS__2: 34cd669195d2ca8c293aa2191417a1d4
867+
12/NAME: 10e10057922963a0ef53b526cb2baf90
868+
12/PRODUCT: 898e5908c9726f8056673258dbe9b1af
869+
12/BONUS: 719f3dbab3cb569a93518d4c2ff1b633
870+
12/BONUS__2: 34cd669195d2ca8c293aa2191417a1d4
871+
13/NAME: 2b0365cf946aeb2678cb2a10f7b662ae
872+
13/PRODUCT: e9c4963b1da635d2365e0111a7a7fc2f
873+
13/BONUS: 35db06a282738c6c6a9417094d39c80e
874+
13/BONUS__2: 34cd669195d2ca8c293aa2191417a1d4
875+
14/NAME: bbe0e79469d425ce26a44bd8c1f9783d
876+
14/PRODUCT: 221ea6f7cf3778ae8b9f079588d7fb7a
877+
14/BONUS: b557d0a6619c27e558b8581ce7d3108a
878+
14/BONUS__2: 34cd669195d2ca8c293aa2191417a1d4
879+
15/NAME: e08118de3644266e79c63d1fe03bdd34
880+
15/PRODUCT: e6b44d2244ead5e4ae5a6a3755d103f8
881+
15/BONUS: 0a91164a59598e2650e4e48eaa6bd4bf
882+
15/BONUS__2: 34cd669195d2ca8c293aa2191417a1d4

‎packages/cli/src/cli/loaders/csv-per-locale.spec.ts‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
import { describe, expect, it } from "vitest";
2+
import createCsvPerLocaleLoader from "./csv-per-locale";
3+
4+
describe("csv-per-locale loader", () => {
5+
const sampleCsv = `id,name,description
6+
35,Hello,Welcome
7+
36,Bye,Farewell`;
8+
9+
const sampleCsvWithDuplicates = `id,name,name,description
10+
35,Hello,Hi,Welcome
11+
36,Bye,Goodbye,Farewell`;
12+
13+
it("pull should parse CSV to array of objects", async () => {
14+
const loader = createCsvPerLocaleLoader();
15+
loader.setDefaultLocale("en");
16+
17+
const result = await loader.pull("en", sampleCsv);
18+
19+
expect(Array.isArray(result)).toBe(true);
20+
expect(result).toHaveLength(2);
21+
expect(result[0]).toEqual({
22+
id: "35",
23+
name: "Hello",
24+
description: "Welcome",
25+
});
26+
expect(result[1]).toEqual({
27+
id: "36",
28+
name: "Bye",
29+
description: "Farewell",
30+
});
31+
});
32+
33+
it("pull should handle duplicate headers by deduplicating", async () => {
34+
const loader = createCsvPerLocaleLoader();
35+
loader.setDefaultLocale("en");
36+
37+
const result = await loader.pull("en", sampleCsvWithDuplicates);
38+
39+
expect(Array.isArray(result)).toBe(true);
40+
expect(result).toHaveLength(2);
41+
expect(result[0]).toEqual({
42+
id: "35",
43+
name: "Hello",
44+
name__2: "Hi",
45+
description: "Welcome",
46+
});
47+
expect(result[1]).toEqual({
48+
id: "36",
49+
name: "Bye",
50+
name__2: "Goodbye",
51+
description: "Farewell",
52+
});
53+
});
54+
55+
it("pull should return empty object for empty CSV", async () => {
56+
const loader = createCsvPerLocaleLoader();
57+
loader.setDefaultLocale("en");
58+
59+
const result = await loader.pull("en", "");
60+
expect(result).toEqual({});
61+
});
62+
63+
it("push should serialize array back to CSV preserving original headers", async () => {
64+
const loader = createCsvPerLocaleLoader();
65+
loader.setDefaultLocale("en");
66+
const originalInput = sampleCsv;
67+
await loader.pull("en", originalInput);
68+
69+
const data = [
70+
{ id: "35", name: "Hello edited", description: "Welcome edited" },
71+
{ id: "36", name: "Bye edited", description: "Farewell edited" },
72+
];
73+
74+
// @ts-expect-error - originalInput is used internally but not in public interface
75+
const csv = await loader.push("en", data, originalInput);
76+
77+
expect(csv).toContain("id,name,description");
78+
expect(csv).toContain("35,Hello edited,Welcome edited");
79+
expect(csv).toContain("36,Bye edited,Farewell edited");
80+
});
81+
82+
it("push should preserve all columns from original CSV including duplicates", async () => {
83+
const loader = createCsvPerLocaleLoader();
84+
loader.setDefaultLocale("en");
85+
const originalInput = sampleCsvWithDuplicates;
86+
await loader.pull("en", originalInput);
87+
88+
const data = [
89+
{ id: "35", name: "Hola", name__2: "Hola2", description: "Bienvenido" },
90+
{ id: "36", name: "Adiós", name__2: "Adiós2", description: "Despedida" },
91+
];
92+
93+
// @ts-expect-error - originalInput is used internally but not in public interface
94+
const csv = await loader.push("en", data, originalInput);
95+
96+
expect(csv).toContain("id,name,name,description");
97+
expect(csv).toContain("35,Hola,Hola2,Bienvenido");
98+
expect(csv).toContain("36,Adiós,Adiós2,Despedida");
99+
});
100+
101+
it("push should handle object with numeric keys", async () => {
102+
const loader = createCsvPerLocaleLoader();
103+
loader.setDefaultLocale("en");
104+
const originalInput = sampleCsv;
105+
await loader.pull("en", originalInput);
106+
107+
const data = {
108+
0: { id: "35", name: "Hola", description: "Bienvenido" },
109+
1: { id: "36", name: "Adiós", description: "Despedida" },
110+
};
111+
112+
// @ts-expect-error - originalInput is used internally but not in public interface
113+
const csv = await loader.push("en", data, originalInput);
114+
115+
expect(csv).toContain("id,name,description");
116+
expect(csv).toContain("35,Hola,Bienvenido");
117+
expect(csv).toContain("36,Adiós,Despedida");
118+
});
119+
});

‎packages/cli/src/cli/loaders/csv-per-locale.ts‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import { ILoader } from "./_types";
2+
import { createLoader } from "./_utils";
3+
import { parse } from "csv-parse/sync";
4+
import { stringify } from "csv-stringify/sync";
5+
6+
function dedupeHeaders(headers: string[]) {
7+
const seen = new Map<string, number>();
8+
9+
return headers.map((h) => {
10+
const count = seen.get(h) ?? 0;
11+
seen.set(h, count + 1);
12+
return count === 0 ? h : `${h}__${count + 1}`;
13+
});
14+
}
15+
16+
export default function createCsvPerLocaleLoader(): ILoader<
17+
string,
18+
Record<string, any>
19+
> {
20+
return createLoader({
21+
async pull(_locale, input) {
22+
if (!input?.trim()) return {};
23+
24+
25+
const parsed = parse(input, {
26+
skip_empty_lines: true,
27+
columns: (headers: string[]) => {
28+
const dedupedHeaders = dedupeHeaders(headers);
29+
return dedupedHeaders;
30+
},
31+
}) as Array<Record<string, any>>;
32+
33+
if (parsed.length === 0) return {};
34+
35+
return parsed;
36+
},
37+
async push(_locale, data, originalInput) {
38+
39+
const rawRows = parse(originalInput || "", {
40+
skip_empty_lines: true,
41+
}) as string[][];
42+
43+
const originalHeaders = rawRows[0];
44+
45+
const dedupedHeaders = dedupeHeaders(originalHeaders);
46+
47+
const columns = originalHeaders.map((header, i) => ({
48+
key: dedupedHeaders[i],
49+
header,
50+
}));
51+
52+
const rows = Object.values(data).filter(
53+
(row) =>
54+
row &&
55+
Object.values(row).some(
56+
(v) => v !== undefined && v !== null,
57+
),
58+
);
59+
// const output = stringify(rows, { header: true }).trimEnd();
60+
const output = stringify(rows, {
61+
header: true,
62+
columns,
63+
}).trimEnd();
64+
65+
return output;
66+
},
67+
});
68+
}

0 commit comments

Comments
 (0)