fix: transform object with numeric keys (#466)

* fix: transform object with numeric keys

Flat loader remembers which sections were object and arrays on pull.
It reconstructs original object on push.

* chore: change naming for better clarity

Author: mathio

.changeset/few-trains-pay.md

@@ -0,0 +1,5 @@
+---
+"lingo.dev": patch
+---
+
+transform object with numeric keys

packages/cli/package.json

@@ -37,6 +37,7 @@
     "dev": "tsup --watch",
     "build": "tsc --noEmit && tsup",
     "test": "vitest run",
+    "test:watch": "vitest",
     "clean": "rm -rf build"
   },
   "keywords": [],

packages/cli/src/cli/loaders/flat.spec.ts

@@ -0,0 +1,91 @@
+import { describe, expect, it } from "vitest";
+import { flatten, unflatten } from "flat";
+import {
+  buildDenormalizedKeysMap,
+  denormalizeObjectKeys,
+  mapDeormalizedKeys,
+  normalizeObjectKeys,
+  OBJECT_NUMERIC_KEY_PREFIX,
+} from "./flat";
+
+describe("flat loader helper functions", () => {
+  const inputObj = {
+    messages: {
+      "1": "a",
+      "2": "b",
+    },
+  };
+  const inputArray = {
+    messages: ["a", "b", "c"],
+  };
+
+  describe("denormalizeObjectKeys", () => {
+    it("should denormalize object keys", () => {
+      const output = denormalizeObjectKeys(inputObj);
+      expect(output).toEqual({
+        messages: {
+          [`${OBJECT_NUMERIC_KEY_PREFIX}1`]: "a",
+          [`${OBJECT_NUMERIC_KEY_PREFIX}2`]: "b",
+        },
+      });
+    });
+
+    it("should preserve array", () => {
+      const output = denormalizeObjectKeys(inputArray);
+      expect(output).toEqual({
+        messages: ["a", "b", "c"],
+      });
+    });
+  });
+
+  describe("buildDenormalizedKeysMap", () => {
+    it("should build normalized keys map", () => {
+      const denormalized: Record<string, string> = flatten(denormalizeObjectKeys(inputObj), { delimiter: "/" });
+      const output = buildDenormalizedKeysMap(denormalized);
+      expect(output).toEqual({
+        "messages/1": `messages/${OBJECT_NUMERIC_KEY_PREFIX}1`,
+        "messages/2": `messages/${OBJECT_NUMERIC_KEY_PREFIX}2`,
+      });
+    });
+
+    it("should build keys map array", () => {
+      const denormalized: Record<string, string> = flatten(denormalizeObjectKeys(inputArray), { delimiter: "/" });
+      const output = buildDenormalizedKeysMap(denormalized);
+      expect(output).toEqual({
+        "messages/0": "messages/0",
+        "messages/1": "messages/1",
+        "messages/2": "messages/2",
+      });
+    });
+  });
+
+  describe("normalizeObjectKeys", () => {
+    it("should normalize denormalized object keys", () => {
+      const output = normalizeObjectKeys(denormalizeObjectKeys(inputObj));
+      expect(output).toEqual(inputObj);
+    });
+
+    it("should process array keys", () => {
+      const output = normalizeObjectKeys(denormalizeObjectKeys(inputArray));
+      expect(output).toEqual(inputArray);
+    });
+  });
+
+  describe("mapDeormalizedKeys", () => {
+    it("should map normalized keys", () => {
+      const denormalized: Record<string, string> = flatten(denormalizeObjectKeys(inputObj), { delimiter: "/" });
+      const keyMap = buildDenormalizedKeysMap(denormalized);
+      const flattened: Record<string, string> = flatten(inputObj, { delimiter: "/" });
+      const mapped = mapDeormalizedKeys(flattened, keyMap);
+      expect(mapped).toEqual(denormalized);
+    });
+
+    it("should map array", () => {
+      const denormalized: Record<string, string> = flatten(denormalizeObjectKeys(inputArray), { delimiter: "/" });
+      const keyMap = buildDenormalizedKeysMap(denormalized);
+      const flattened: Record<string, string> = flatten(inputArray, { delimiter: "/" });
+      const mapped = mapDeormalizedKeys(flattened, keyMap);
+      expect(mapped).toEqual(denormalized);
+    });
+  });
+});

packages/cli/src/cli/loaders/flat.ts

@@ -1,24 +1,88 @@
 import { flatten, unflatten } from "flat";
 import { ILoader } from "./_types";
 import { createLoader } from "./_utils";
+import _ from "lodash";
+
+export const OBJECT_NUMERIC_KEY_PREFIX = "__lingodotdev__obj__";
 
 export default function createFlatLoader(): ILoader<Record<string, any>, Record<string, string>> {
+  let denormalizedKeysMap: Record<string, string>;
+
   return createLoader({
     pull: async (locale, input) => {
-      return flatten(input || {}, {
+      const denormalized = denormalizeObjectKeys(input || {});
+      const flattened: Record<string, string> = flatten(denormalized, {
         delimiter: "/",
         transformKey(key) {
           return encodeURIComponent(String(key));
         },
       });
+      denormalizedKeysMap = buildDenormalizedKeysMap(flattened);
+      const normalized = normalizeObjectKeys(flattened);
+      return normalized;
     },
     push: async (locale, data) => {
-      return unflatten(data || {}, {
+      const denormalized = mapDeormalizedKeys(data, denormalizedKeysMap);
+      const unflattened: Record<string, any> = unflatten(denormalized || {}, {
         delimiter: "/",
         transformKey(key) {
           return decodeURIComponent(String(key));
         },
       });
+      const normalized = normalizeObjectKeys(unflattened);
+      return normalized;
     },
   });
 }
+
+export function buildDenormalizedKeysMap(obj: Record<string, string>) {
+  return Object.keys(obj).reduce(
+    (acc, key) => {
+      const normalizedKey = `${key}`.replace(OBJECT_NUMERIC_KEY_PREFIX, "");
+      acc[normalizedKey] = key;
+      return acc;
+    },
+    {} as Record<string, string>,
+  );
+}
+
+export function mapDeormalizedKeys(obj: Record<string, any>, denormalizedKeysMap: Record<string, string>) {
+  return Object.keys(obj).reduce(
+    (acc, key) => {
+      const denormalizedKey = denormalizedKeysMap[key];
+      acc[denormalizedKey] = obj[key];
+      return acc;
+    },
+    {} as Record<string, string>,
+  );
+}
+
+export function denormalizeObjectKeys(obj: Record<string, any>): Record<string, any> {
+  if (_.isObject(obj) && !_.isArray(obj)) {
+    return _.transform(
+      obj,
+      (result, value, key) => {
+        const newKey = !isNaN(Number(key)) ? `${OBJECT_NUMERIC_KEY_PREFIX}${key}` : key;
+        result[newKey] = _.isObject(value) ? denormalizeObjectKeys(value) : value;
+      },
+      {} as Record<string, any>,
+    );
+  } else {
+    return obj;
+  }
+}
+
+export function normalizeObjectKeys(obj: Record<string, any>): Record<string, any> {
+  if (_.isObject(obj) && !_.isArray(obj)) {
+    return _.transform(
+      obj,
+      (result, value, key) => {
+        const newKey = `${key}`.replace(OBJECT_NUMERIC_KEY_PREFIX, "");
+        result[newKey] = _.isObject(value) ? normalizeObjectKeys(value) : value;
+      },
+      {} as Record<string, any>,
+    );
+  } else {
+    return obj;
+  }
+}

packages/cli/src/cli/loaders/index.spec.ts

@@ -332,6 +332,42 @@ describe("bucket loaders", () => {
 
       expect(fs.writeFile).toHaveBeenCalledWith("i18n/es.json", expectedOutput, { encoding: "utf-8", flag: "w" });
     });
+
+    it("should save json data with numeric keys", async () => {
+      setupFileMocks();
+
+      const input = { messages: { "1": "foo", "2": "bar", "3": "bar" } };
+      const payload = { "messages/1": "foo", "messages/2": "bar", "messages/3": "bar" };
+      const expectedOutput = JSON.stringify(input, null, 2);
+
+      mockFileOperations(JSON.stringify(input));
+
+      const jsonLoader = createBucketLoader("json", "i18n/[locale].json");
+      jsonLoader.setDefaultLocale("en");
+      await jsonLoader.pull("en");
+
+      await jsonLoader.push("es", payload);
+
+      expect(fs.writeFile).toHaveBeenCalledWith("i18n/es.json", expectedOutput, { encoding: "utf-8", flag: "w" });
+    });
+
+    it("should save json data with array", async () => {
+      setupFileMocks();
+
+      const input = { messages: ["foo", "bar"] };
+      const payload = { "messages/0": "foo", "messages/1": "bar" };
+      const expectedOutput = `{\n  "messages\": [\"foo\", \"bar\"]\n}`;
+
+      mockFileOperations(JSON.stringify(input));
+
+      const jsonLoader = createBucketLoader("json", "i18n/[locale].json");
+      jsonLoader.setDefaultLocale("en");
+      await jsonLoader.pull("en");
+
+      await jsonLoader.push("es", payload);
+
+      expect(fs.writeFile).toHaveBeenCalledWith("i18n/es.json", expectedOutput, { encoding: "utf-8", flag: "w" });
+    });
   });
 
   describe("markdown bucket loader", () => {