Skip to content

Commit e76f0b4

Browse files
maxprilutskiymaxprilutskiy
authored
feat: key renames (#622)
* feat: delta processor * feat: draft * chore: upd * chore: fix delta calcs * test: add tests
1 parent 730968a commit e76f0b4

11 files changed

Lines changed: 471 additions & 642 deletions

File tree

‎packages/cli/demo/json/en.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"home": {
3-
"title/main": "Lingo.dev",
3+
"title/main": "Hello, Lingo.dev!",
44
"description/dev": "Lingo.dev is an AI-powered localization-as-a-service platform for modern SaaS.",
55
"i-am-a-developer": "I'm a software developer that created this website."
66
}

‎packages/cli/demo/json/es.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"home": {
3-
"title/main": "Lingo.dev",
3+
"title/main": "¡Hola, Lingo.dev!",
44
"description/dev": "Lingo.dev es una plataforma de localización como servicio impulsada por IA para SaaS moderno.",
55
"i-am-a-developer": "Soy un desarrollador de software que creó este sitio web."
66
}
7-
}
7+
}

‎packages/cli/i18n.json‎

Lines changed: 1 addition & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -7,51 +7,11 @@
77
},
88
"locale": {
99
"source": "en",
10-
"targets": ["es", "fr", "it", "de", "ar"]
10+
"targets": ["es"]
1111
},
1212
"buckets": {
13-
"dato": {
14-
"include": ["demo/dato/page.json5"]
15-
},
16-
"android": {
17-
"include": ["demo/android/[locale].xml"]
18-
},
19-
"csv": {
20-
"include": ["demo/csv/file.csv"]
21-
},
22-
"flutter": {
23-
"include": ["demo/flutter/[locale].arb"]
24-
},
25-
"html": {
26-
"include": ["demo/html/[locale].html"]
27-
},
2813
"json": {
2914
"include": ["demo/json/[locale].json"]
30-
},
31-
"markdown": {
32-
"include": ["demo/markdown/[locale]/*.md"],
33-
"exclude": ["demo/markdown/[locale]/ignored.md"]
34-
},
35-
"po": {
36-
"include": ["demo/po/[locale].po"]
37-
},
38-
"properties": {
39-
"include": ["demo/properties/[locale].properties"]
40-
},
41-
"xcode-strings": {
42-
"include": ["demo/xcode-strings/[locale].lproj/Localizable.strings"]
43-
},
44-
"xcode-stringsdict": {
45-
"include": ["demo/xcode-stringsdict/[locale].lproj/Localizable.stringsdict"]
46-
},
47-
"xcode-xcstrings": {
48-
"include": ["demo/xcode-xcstrings/Localizable.xcstrings"]
49-
},
50-
"yaml": {
51-
"include": ["demo/yaml/[locale].yml"]
52-
},
53-
"yaml-root-key": {
54-
"include": ["demo/yaml-root-key/[locale].yml"]
5515
}
5616
},
5717
"$schema": "https://lingo.dev/schema/i18n.json"

‎packages/cli/i18n.lock‎

Lines changed: 1 addition & 579 deletions
Large diffs are not rendered by default.

‎packages/cli/src/cli/cmd/i18n.ts‎

Lines changed: 97 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,12 @@ import { bucketTypeSchema, I18nConfig, localeCodeSchema, resolveOverriddenLocale
22
import { Command } from "interactive-commander";
33
import Z from "zod";
44
import _ from "lodash";
5+
import * as path from "path";
56
import { getConfig } from "../utils/config";
67
import { getSettings } from "../utils/settings";
78
import { CLIError } from "../utils/errors";
89
import Ora from "ora";
910
import createBucketLoader from "../loaders";
10-
import { createLockfileHelper } from "../utils/lockfile";
1111
import { createAuthenticator } from "../utils/auth";
1212
import { getBuckets } from "../utils/buckets";
1313
import chalk from "chalk";
@@ -19,6 +19,9 @@ import updateGitignore from "../utils/update-gitignore";
1919
import createProcessor from "../processor";
2020
import { withExponentialBackoff } from "../utils/exp-backoff";
2121
import trackEvent from "../utils/observability";
22+
import { createDeltaProcessor } from "../utils/delta";
23+
import { tryReadFile, writeFile } from "../utils/fs";
24+
import { flatten, unflatten } from "flat";
2225

2326
export default new Command()
2427
.command("i18n")
@@ -111,16 +114,16 @@ export default new Command()
111114
}
112115

113116
const targetLocales = flags.locale?.length ? flags.locale : i18nConfig!.locale.targets;
114-
const lockfileHelper = createLockfileHelper();
115117

116118
// Ensure the lockfile exists
117-
ora.start("Ensuring i18n.lock exists...");
118-
if (!lockfileHelper.isLockfileExists()) {
119+
ora.start("Setting up localization cache...");
120+
const checkLockfileProcessor = createDeltaProcessor("");
121+
const lockfileExists = await checkLockfileProcessor.checkIfLockExists();
122+
if (!lockfileExists) {
119123
ora.start("Creating i18n.lock...");
120124
for (const bucket of buckets) {
121125
for (const bucketPath of bucket.paths) {
122126
const sourceLocale = resolveOverriddenLocale(i18nConfig!.locale.source, bucketPath.delimiter);
123-
124127
const bucketLoader = createBucketLoader(bucket.type, bucketPath.pathPattern, {
125128
isCacheRestore: false,
126129
defaultLocale: sourceLocale,
@@ -130,12 +133,73 @@ export default new Command()
130133
await bucketLoader.init();
131134

132135
const sourceData = await bucketLoader.pull(i18nConfig!.locale.source);
133-
lockfileHelper.registerSourceData(bucketPath.pathPattern, sourceData);
136+
137+
const deltaProcessor = createDeltaProcessor(bucketPath.pathPattern);
138+
const checksums = await deltaProcessor.createChecksums(sourceData);
139+
await deltaProcessor.saveChecksums(checksums);
134140
}
135141
}
136-
ora.succeed("i18n.lock created");
142+
ora.succeed("Localization cache initialized");
137143
} else {
138-
ora.succeed("i18n.lock loaded");
144+
ora.succeed("Localization cache loaded");
145+
}
146+
// Handle json key renames
147+
for (const bucket of buckets) {
148+
if (bucket.type !== "json") {
149+
continue;
150+
}
151+
ora.start("Validating localization state...");
152+
for (const bucketPath of bucket.paths) {
153+
const sourceLocale = resolveOverriddenLocale(i18nConfig!.locale.source, bucketPath.delimiter);
154+
const deltaProcessor = createDeltaProcessor(bucketPath.pathPattern);
155+
const sourcePath = path.join(process.cwd(), bucketPath.pathPattern.replace("[locale]", sourceLocale));
156+
const sourceContent = tryReadFile(sourcePath, null);
157+
const sourceData = JSON.parse(sourceContent || "{}");
158+
const sourceFlattenedData = flatten(sourceData, {
159+
delimiter: "/",
160+
transformKey(key) {
161+
return encodeURIComponent(key);
162+
},
163+
}) as Record<string, any>;
164+
165+
for (const _targetLocale of targetLocales) {
166+
const targetLocale = resolveOverriddenLocale(_targetLocale, bucketPath.delimiter);
167+
const targetPath = path.join(process.cwd(), bucketPath.pathPattern.replace("[locale]", targetLocale));
168+
const targetContent = tryReadFile(targetPath, null);
169+
const targetData = JSON.parse(targetContent || "{}");
170+
const targetFlattenedData = flatten(targetData, {
171+
delimiter: "/",
172+
transformKey(key) {
173+
return encodeURIComponent(key);
174+
},
175+
}) as Record<string, any>;
176+
177+
const checksums = await deltaProcessor.loadChecksums();
178+
const delta = await deltaProcessor.calculateDelta({
179+
sourceData: sourceFlattenedData,
180+
targetData: targetFlattenedData,
181+
checksums,
182+
});
183+
if (!delta.hasChanges) {
184+
continue;
185+
}
186+
187+
for (const [oldKey, newKey] of delta.renamed) {
188+
targetFlattenedData[newKey] = targetFlattenedData[oldKey];
189+
delete targetFlattenedData[oldKey];
190+
}
191+
192+
const updatedTargetData = unflatten(targetFlattenedData, {
193+
delimiter: "/",
194+
transformKey(key) {
195+
return decodeURIComponent(key);
196+
},
197+
}) as Record<string, any>;
198+
199+
await writeFile(targetPath, JSON.stringify(updatedTargetData, null, 2));
200+
}
201+
}
202+
ora.succeed("Localization state check completed");
139203
}
140204

141205
// recover cache if exists
@@ -175,7 +239,9 @@ export default new Command()
175239
}
176240

177241
await bucketLoader.push(targetLocale, targetData);
178-
lockfileHelper.registerPartialSourceData(bucketPath.pathPattern, cachedSourceData);
242+
const deltaProcessor = createDeltaProcessor(bucketPath.pathPattern);
243+
const checksums = await deltaProcessor.createChecksums(cachedSourceData);
244+
await deltaProcessor.saveChecksums(checksums);
179245

180246
bucketOra.succeed(
181247
`[${sourceLocale} -> ${targetLocale}] Recovered ${Object.keys(cachedSourceData).length} entries from cache`,
@@ -210,7 +276,15 @@ export default new Command()
210276
const { unlocalizable: sourceUnlocalizable, ...sourceData } = await bucketLoader.pull(
211277
i18nConfig!.locale.source,
212278
);
213-
const updatedSourceData = lockfileHelper.extractUpdatedData(bucketPath.pathPattern, sourceData);
279+
const deltaProcessor = createDeltaProcessor(bucketPath.pathPattern);
280+
const sourceChecksums = await deltaProcessor.createChecksums(sourceData);
281+
const savedChecksums = await deltaProcessor.loadChecksums();
282+
283+
// Get updated data by comparing current checksums with saved checksums
284+
const updatedSourceData = _.pickBy(
285+
sourceData,
286+
(value, key) => sourceChecksums[key] !== savedChecksums[key],
287+
);
214288

215289
// translation was updated in the source file
216290
if (Object.keys(updatedSourceData).length > 0) {
@@ -288,16 +362,20 @@ export default new Command()
288362

289363
sourceData = await bucketLoader.pull(sourceLocale);
290364

291-
const updatedSourceData = flags.force
292-
? sourceData
293-
: lockfileHelper.extractUpdatedData(bucketPath.pathPattern, sourceData);
294-
295365
const targetData = await bucketLoader.pull(targetLocale);
296-
let processableData = calculateDataDelta({
366+
const deltaProcessor = createDeltaProcessor(bucketPath.pathPattern);
367+
const checksums = await deltaProcessor.loadChecksums();
368+
const delta = await deltaProcessor.calculateDelta({
297369
sourceData,
298-
updatedSourceData,
299370
targetData,
371+
checksums,
300372
});
373+
let processableData = _.chain(sourceData)
374+
.entries()
375+
.filter(([key, value]) => delta.added.includes(key) || delta.updated.includes(key) || !!flags.force)
376+
.fromPairs()
377+
.value();
378+
301379
if (flags.key) {
302380
processableData = _.pickBy(processableData, (_, key) => key === flags.key);
303381
}
@@ -382,7 +460,9 @@ export default new Command()
382460
}
383461
}
384462

385-
lockfileHelper.registerSourceData(bucketPath.pathPattern, sourceData);
463+
const deltaProcessor = createDeltaProcessor(bucketPath.pathPattern);
464+
const checksums = await deltaProcessor.createChecksums(sourceData);
465+
await deltaProcessor.saveChecksums(checksums);
386466
}
387467
} catch (_error: any) {
388468
const error = new Error(`Failed to process bucket ${bucket.type}: ${_error.message}`);

packages/cli/src/cli/processor/openai.ts renamed to packages/cli/src/cli/processor/basic.ts

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@ import { LocalizerInput, LocalizerProgressFn } from "./_base";
33

44
export function createBasicTranslator(model: LanguageModelV1, systemPrompt: string) {
55
return async (input: LocalizerInput, onProgress: LocalizerProgressFn) => {
6+
if (!Object.keys(input.processableData).length) {
7+
return input.processableData;
8+
}
9+
610
if (!process.env.OPENAI_API_KEY) {
711
throw new Error("OPENAI_API_KEY is not set");
812
}
@@ -50,6 +54,6 @@ export function createBasicTranslator(model: LanguageModelV1, systemPrompt: stri
5054

5155
const result = JSON.parse(response.text);
5256

53-
return result;
57+
return result?.data || {};
5458
};
5559
}

‎packages/cli/src/cli/processor/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { I18nConfig } from "@lingo.dev/_spec";
22
import { LocalizerFn } from "./_base";
33
import { createLingoLocalizer } from "./lingo";
4-
import { createBasicTranslator } from "./openai";
4+
import { createBasicTranslator } from "./basic";
55
import { createOpenAI } from "@ai-sdk/openai";
66
import { createAnthropic } from "@ai-sdk/anthropic";
77

‎packages/cli/src/cli/processor/lingo.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@ import { LocalizerInput, LocalizerProgressFn } from "./_base";
33

44
export function createLingoLocalizer(params: { apiKey: string; apiUrl: string }) {
55
return async (input: LocalizerInput, onProgress: LocalizerProgressFn) => {
6+
if (!Object.keys(input.processableData).length) {
7+
return input.processableData;
8+
}
9+
610
const lingo = new LingoDotDevEngine({
711
apiKey: params.apiKey,
812
apiUrl: params.apiUrl,

0 commit comments

Comments
 (0)