Skip to content

Commit 3c40593

Browse files
authored
feat(new-compiler): migrate metadata storage from lockfile to LMDB (#1955)
* feat(new-compiler): migrate metadata storage from lockfile to LMDB
1 parent cc95229 commit 3c40593

File tree

18 files changed

+572
-319
lines changed

18 files changed

+572
-319
lines changed

.changeset/common-teeth-reply.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
---
2+
"@lingo.dev/compiler": patch
3+
---
4+
5+
- Migrate metadata storage from JSON files to LMDB
6+
- New storage locations: .lingo/metadata-dev/ and .lingo/metadata-build/
7+
- Update new-compiler docs
8+
- Remove proper-lockfile dependency

packages/new-compiler/README.md

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -447,9 +447,10 @@ The compiler is organized into several key modules:
447447

448448
#### `src/metadata/` - Translation metadata management
449449

450-
- **`manager.ts`** - CRUD operations for `.lingo/metadata.json`
451-
- Thread-safe metadata file operations with file locking
450+
- **`manager.ts`** - CRUD operations for LMDB metadata database
451+
- Uses LMDB for high-performance key-value storage with built-in concurrency
452452
- Manages translation entries with hash-based identifiers
453+
- Stores metadata in `.lingo/metadata-dev/` (development) or `.lingo/metadata-build/` (production)
453454

454455
#### `src/translators/` - Translation provider abstraction
455456

packages/new-compiler/docs/TRANSLATION_ARCHITECTURE.md

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ metadata management, translation execution, and caching.
77

88
## Architectural Principles
99

10-
1. **Metadata file structure** is only known by:
11-
- Metadata Manager (reads/writes metadata.json)
10+
1. **Metadata storage** is only known by:
11+
- Metadata functions (reads/writes LMDB database)
1212
- Translation Service (orchestrator that coordinates everything)
1313

1414
2. **Translators are stateless** and work with abstract `TranslatableEntry` types
@@ -35,12 +35,12 @@ metadata management, translation execution, and caching.
3535
└────────────────┬────────────────────────────────┘
3636
│ writes
3737
38-
┌─────────────────────────────────────────────────
39-
MetadataManager
40-
│ - ONLY component that reads/writes metadata.json
41-
│ - Provides metadata loading/saving
42-
│ - Returns TranslationEntry[]
43-
└────────────────┬────────────────────────────────
38+
┌─────────────────────────────────────────────────┐
39+
Metadata Functions (saveMetadata/loadMetadata)
40+
│ - Pure functions for LMDB database access
41+
│ - Per-operation connections (see manager.ts)
42+
│ - Returns TranslationEntry[] │
43+
└────────────────┬────────────────────────────────┘
4444
│ reads from
4545
4646
┌─────────────────────────────────────────────────┐

packages/new-compiler/package.json

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,6 @@
143143
"@types/babel__traverse": "7.28.0",
144144
"@types/ini": "4.1.1",
145145
"@types/node": "25.0.3",
146-
"@types/proper-lockfile": "4.1.4",
147146
"@types/react": "19.2.7",
148147
"@types/react-dom": "19.2.3",
149148
"@types/ws": "8.18.1",
@@ -178,7 +177,7 @@
178177
"lodash": "4.17.21",
179178
"node-machine-id": "1.1.12",
180179
"posthog-node": "5.14.0",
181-
"proper-lockfile": "4.1.2",
180+
"lmdb": "3.2.6",
182181
"ws": "8.18.3"
183182
},
184183
"peerDependencies": {
Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
import { describe, it, expect, beforeEach, afterEach } from "vitest";
2+
import fs from "fs";
3+
import path from "path";
4+
import os from "os";
5+
import {
6+
loadMetadata,
7+
saveMetadata,
8+
cleanupExistingMetadata,
9+
getMetadataPath,
10+
} from "./manager";
11+
import type { ContentTranslationEntry } from "../types";
12+
import { generateTranslationHash } from "../utils/hash";
13+
14+
function createTestEntry(sourceText: string): ContentTranslationEntry {
15+
const context = { filePath: "test.tsx", componentName: "TestComponent" };
16+
return {
17+
type: "content",
18+
sourceText,
19+
context,
20+
location: { filePath: "test.tsx", line: 1, column: 1 },
21+
hash: generateTranslationHash(sourceText, context),
22+
};
23+
}
24+
25+
function createUniqueDbPath(): string {
26+
return path.join(
27+
os.tmpdir(),
28+
`lmdb-test-${Date.now()}-${Math.random().toString(36).slice(2)}`,
29+
);
30+
}
31+
32+
describe("metadata", () => {
33+
let testDbPath: string;
34+
35+
beforeEach(() => {
36+
testDbPath = createUniqueDbPath();
37+
});
38+
39+
afterEach(() => {
40+
cleanupExistingMetadata(testDbPath);
41+
});
42+
43+
describe("loadMetadata", () => {
44+
it("should return empty metadata for new database", async () => {
45+
const metadata = await loadMetadata(testDbPath);
46+
expect(metadata).toEqual({});
47+
});
48+
49+
it("should load and preserve all entry fields", async () => {
50+
const entry = createTestEntry("Hello world");
51+
52+
await saveMetadata(testDbPath, [entry]);
53+
const metadata = await loadMetadata(testDbPath);
54+
55+
expect(metadata[entry.hash]).toEqual(entry);
56+
expect(Object.keys(metadata).length).toBe(1);
57+
});
58+
59+
it("should handle entries with very long sourceText", async () => {
60+
const longText = "A".repeat(100000);
61+
const entry = createTestEntry(longText);
62+
await saveMetadata(testDbPath, [entry]);
63+
64+
const metadata = await loadMetadata(testDbPath);
65+
expect(metadata[entry.hash].sourceText).toBe(longText);
66+
});
67+
});
68+
69+
describe("saveMetadata", () => {
70+
it("should save a single entry", async () => {
71+
const entry = createTestEntry("v1");
72+
await saveMetadata(testDbPath, [entry]);
73+
const metadata = await loadMetadata(testDbPath);
74+
expect(Object.keys(metadata).length).toBe(1);
75+
expect(metadata[entry.hash].sourceText).toBe("v1");
76+
});
77+
78+
it("should accumulate entries across saves", async () => {
79+
await saveMetadata(testDbPath, [createTestEntry("text-1")]);
80+
await saveMetadata(testDbPath, [
81+
createTestEntry("text-2"),
82+
createTestEntry("text-3"),
83+
]);
84+
const metadata = await loadMetadata(testDbPath);
85+
expect(Object.keys(metadata).length).toBe(3);
86+
});
87+
88+
it("should overwrite existing entry on re-save", async () => {
89+
const entry = createTestEntry("text-1");
90+
await saveMetadata(testDbPath, [entry]);
91+
92+
const updatedEntry = createTestEntry("text-1");
93+
updatedEntry.location = { filePath: "moved.tsx", line: 99, column: 5 };
94+
await saveMetadata(testDbPath, [updatedEntry]);
95+
96+
const updated = await loadMetadata(testDbPath);
97+
expect(Object.keys(updated).length).toBe(1);
98+
expect(updated[entry.hash].location.filePath).toBe("moved.tsx");
99+
});
100+
101+
it("should handle empty array save", async () => {
102+
await saveMetadata(testDbPath, []);
103+
const metadata = await loadMetadata(testDbPath);
104+
expect(Object.keys(metadata).length).toBe(0);
105+
});
106+
107+
it("should handle large batch of entries", async () => {
108+
const entries = Array.from({ length: 100 }, (_, i) =>
109+
createTestEntry(`batch-${i}`),
110+
);
111+
112+
await saveMetadata(testDbPath, entries);
113+
expect(Object.keys(await loadMetadata(testDbPath)).length).toBe(100);
114+
});
115+
});
116+
117+
describe("concurrent access (single process)", () => {
118+
it("should handle 1000 concurrent operations", async () => {
119+
const promises = Array.from({ length: 1000 }, async (_, i) => {
120+
await saveMetadata(testDbPath, [createTestEntry(`entry-${i}`)]);
121+
});
122+
await Promise.all(promises);
123+
124+
const final = await loadMetadata(testDbPath);
125+
expect(Object.keys(final).length).toBe(1000);
126+
});
127+
});
128+
129+
describe("cleanupExistingMetadata", () => {
130+
it("should remove database and allow reopening with fresh state", async () => {
131+
const entry = createTestEntry("before");
132+
await saveMetadata(testDbPath, [entry]);
133+
expect(fs.existsSync(testDbPath)).toBe(true);
134+
135+
// Cleanup should succeed because saveMetadata closes the DB
136+
cleanupExistingMetadata(testDbPath);
137+
expect(fs.existsSync(testDbPath)).toBe(false);
138+
139+
// Should work with fresh state after cleanup
140+
const metadata = await loadMetadata(testDbPath);
141+
expect(metadata[entry.hash]).toBeUndefined();
142+
expect(Object.keys(metadata).length).toBe(0);
143+
});
144+
145+
it("should handle non-existent path and multiple calls gracefully", () => {
146+
const nonExistent = path.join(os.tmpdir(), "does-not-exist-db");
147+
expect(() => cleanupExistingMetadata(nonExistent)).not.toThrow();
148+
expect(() => cleanupExistingMetadata(nonExistent)).not.toThrow();
149+
});
150+
});
151+
152+
describe("getMetadataPath", () => {
153+
it("should return correct path based on environment and config", () => {
154+
const devResult = getMetadataPath({
155+
sourceRoot: "/app",
156+
lingoDir: ".lingo",
157+
environment: "development",
158+
});
159+
expect(devResult).toContain("metadata-dev");
160+
expect(devResult).not.toContain("metadata-build");
161+
162+
const prodResult = getMetadataPath({
163+
sourceRoot: "/app",
164+
lingoDir: ".lingo",
165+
environment: "production",
166+
});
167+
expect(prodResult).toContain("metadata-build");
168+
169+
const customResult = getMetadataPath({
170+
sourceRoot: "/app",
171+
lingoDir: ".custom-lingo",
172+
environment: "development",
173+
});
174+
expect(customResult).toContain(".custom-lingo");
175+
});
176+
});
177+
178+
describe("error handling", () => {
179+
it("should throw error for invalid path", async () => {
180+
const invalidPath = "/root/definitely/cannot/create/this/path";
181+
await expect(loadMetadata(invalidPath)).rejects.toThrow(
182+
`Failed to open LMDB at ${invalidPath}`,
183+
);
184+
});
185+
});
186+
});

0 commit comments

Comments
 (0)