✨(tokens) add token references

Previously we were not fully using CSS variables as values used in
CSS were hard-coded one. It wasn't keeping the variable references.
This was causing issues when customizing the theme, because editing
colors was not enough, it was needed to customize also the tokens
using these variables. Now by introducing ref() we can delegate how
to deal with these directly to the generators themselves.

.changeset/chatty-garlics-exercise.md

@@ -0,0 +1,5 @@
+---
+"@openfun/cunningham-tokens": minor
+---
+
+add token references

packages/tokens/jest.config.ts

@@ -3,7 +3,7 @@ import type { JestConfigWithTsJest } from "ts-jest";
 const jestConfig: JestConfigWithTsJest = {
   preset: "ts-jest",
   testEnvironment: "node",
-  moduleDirectories: ["node_modules", "src/bin"],
+  moduleDirectories: ["node_modules", "src/bin", "src/lib"],
   setupFiles: ["<rootDir>/src/bin/tests/Setup.ts"],
   transform: {
     "^.+\\.tsx?$": [

packages/tokens/src/bin/Generators/CssGenerator.ts

@@ -1,11 +1,20 @@
 import * as path from "path";
 import { flatify } from "Utils/Flatify";
 import Config from "Config";
-import { Generator } from "Generators/index";
+import { Generator, resolveRefs } from "Generators/index";
 import { put } from "Utils/Files";
 import { Tokens } from "TokensGenerator";
 
 export const cssGenerator: Generator = async (tokens, opts) => {
+  // Replace refs by CSS variables.
+  tokens = resolveRefs(tokens, (ref) => {
+    const cssVar =
+      "--" +
+      Config.sass.varPrefix +
+      ref.replaceAll(".", Config.sass.varSeparator);
+    return `var(${cssVar})`;
+  });
+
   const flatTokens = flatify(tokens, Config.sass.varSeparator);
   const cssVars = Object.keys(flatTokens).reduce((acc, token) => {
     return (

packages/tokens/src/bin/Generators/JsGenerator.spec.ts

@@ -24,7 +24,7 @@ describe("JsGenerator", () => {
     await run(["", "", "-g", "js"]);
     expect(fs.existsSync(tokensFile)).toEqual(true);
     expect(fs.readFileSync(tokensFile).toString()).toMatchInlineSnapshot(`
-      "export const tokens = {"theme":{"colors":{"primary":"#055FD2","secondary":"#DA0000","ternary-900":"#022858","ogre-odor-is-orange-indeed":"#FD5240"},"font":{"sizes":{"m":"1rem"},"weights":{"medium":400},"families":{"base":"Roboto"}},"spacings":{"s":"1rem"},"transitions":{"ease":"linear"}}};
+      "export const tokens = {"theme":{"colors":{"primary":"#055FD2","secondary":"#DA0000","ternary-900":"#022858","ogre-odor-is-orange-indeed":"#FD5240"},"font":{"sizes":{"m":"1rem"},"weights":{"medium":400},"families":{"base":"Roboto"}},"spacings":{"s":"1rem"},"transitions":{"ease":"linear"},"input":{"border-color":"#022858"}}};
       "
     `);
   });

packages/tokens/src/bin/Generators/JsGenerator.ts

@@ -1,10 +1,11 @@
 import path from "path";
-import { Generator } from "Generators/index";
+import { Generator, resolveRefs, resolveRefValue } from "Generators/index";
 import Config from "Config";
 import { put } from "Utils/Files";
 import { Tokens } from "TokensGenerator";
 
 export const jsGenerator: Generator = async (tokens, opts) => {
+  tokens = resolveRefs(tokens, resolveRefValue);
   const dest = path.join(opts.path, Config.tokenFilename + ".js");
   put(dest, await jsGeneratorContent(tokens));
 };

packages/tokens/src/bin/Generators/SassGenerator.ts

@@ -1,10 +1,11 @@
 import path from "path";
-import { Generator } from "Generators/index";
+import { Generator, resolveRefs, resolveRefValue } from "Generators/index";
 import Config from "Config";
 import { put } from "Utils/Files";
 import { Tokens } from "TokensGenerator";
 
 export const sassGenerator: Generator = async (tokens, opts) => {
+  tokens = resolveRefs(tokens, resolveRefValue);
   const sassContent = generateSassMaps(tokens);
   const outputPath = path.join(opts.path, Config.tokenFilename + ".scss");
   put(outputPath, sassContent);

packages/tokens/src/bin/Generators/TsGenerator.spec.ts

@@ -24,7 +24,7 @@ describe("TsGenerator", () => {
     await run(["", "", "-g", "ts"]);
     expect(fs.existsSync(tokensFile)).toEqual(true);
     expect(fs.readFileSync(tokensFile).toString()).toMatchInlineSnapshot(`
-      "export const tokens = {"theme":{"colors":{"primary":"#055FD2","secondary":"#DA0000","ternary-900":"#022858","ogre-odor-is-orange-indeed":"#FD5240"},"font":{"sizes":{"m":"1rem"},"weights":{"medium":400},"families":{"base":"Roboto"}},"spacings":{"s":"1rem"},"transitions":{"ease":"linear"}}};
+      "export const tokens = {"theme":{"colors":{"primary":"#055FD2","secondary":"#DA0000","ternary-900":"#022858","ogre-odor-is-orange-indeed":"#FD5240"},"font":{"sizes":{"m":"1rem"},"weights":{"medium":400},"families":{"base":"Roboto"}},"spacings":{"s":"1rem"},"transitions":{"ease":"linear"},"input":{"border-color":"#022858"}}};
       "
     `);
   });

packages/tokens/src/bin/Generators/TsGenerator.ts

@@ -1,10 +1,11 @@
 import path from "path";
-import { Generator } from "Generators/index";
+import { Generator, resolveRefs, resolveRefValue } from "Generators/index";
 import Config from "Config";
 import { put } from "Utils/Files";
 import { jsGeneratorContent } from "Generators/JsGenerator";
 
 export const tsGenerator: Generator = async (tokens, opts) => {
+  tokens = resolveRefs(tokens, resolveRefValue);
   const dest = path.join(opts.path, Config.tokenFilename + ".ts");
   put(dest, await jsGeneratorContent(tokens));
 };

packages/tokens/src/bin/Generators/index.spec.ts

@@ -0,0 +1,103 @@
+import { resolveRefs } from "Generators/index";
+import { resolve } from "Utils/resolve";
+import { Tokens } from "TokensGenerator";
+
+describe("resolveRefs", () => {
+  it("should replace root level refs", async () => {
+    const tokens = {
+      a: "b",
+      c: "ref(a)",
+    };
+    const res = resolveRefs(
+      tokens as unknown as Tokens,
+      (ref, resolvingTokens) => {
+        return resolve(resolvingTokens, ref);
+      }
+    );
+    expect(res).toEqual({
+      a: "b",
+      c: "b",
+    });
+  });
+  it("should replace root level refs with custom callback returning same value", async () => {
+    const tokens = {
+      a: "b",
+      c: "ref(a)",
+      d: "ref(zzzz)",
+    };
+    const res = resolveRefs(tokens as unknown as Tokens, () => {
+      return "idem";
+    });
+    expect(res).toEqual({
+      a: "b",
+      c: "idem",
+      d: "idem",
+    });
+  });
+
+  it("should replace nested refs", async () => {
+    const tokens = {
+      a: "b",
+      c: {
+        c1: {
+          c2: "value",
+        },
+      },
+      d: {
+        d1: "ref(c.c1.c2)",
+      },
+    };
+    const res = resolveRefs(
+      tokens as unknown as Tokens,
+      (ref, resolvingTokens) => {
+        return resolve(resolvingTokens, ref);
+      }
+    );
+    expect(res).toEqual({
+      a: "b",
+      c: {
+        c1: {
+          c2: "value",
+        },
+      },
+      d: {
+        d1: "value",
+      },
+    });
+  });
+  it("should handle transitive refs", async () => {
+    const tokens = {
+      a: "value",
+      b: "ref(a)",
+      c: "ref(b)",
+      d: "ref(c)",
+      e: "ref(d)",
+    };
+    const res = resolveRefs(
+      tokens as unknown as Tokens,
+      (ref, resolvingTokens) => {
+        return resolve(resolvingTokens, ref);
+      }
+    );
+    expect(res).toEqual({
+      a: "value",
+      b: "value",
+      c: "value",
+      d: "value",
+      e: "value",
+    });
+  });
+  it("should throw on maximum iterations", async () => {
+    const tokens = {
+      a: "value",
+      b: "ref(a)",
+    };
+    expect(() => {
+      resolveRefs(tokens as unknown as Tokens, (ref) => {
+        return `ref(${ref})`;
+      });
+    }).toThrow(
+      "Maximum resolveRefs iterations: please reduce usage of chained references."
+    );
+  });
+});

packages/tokens/src/bin/Generators/index.ts

@@ -3,6 +3,7 @@ import { sassGenerator } from "Generators/SassGenerator";
 import { jsGenerator } from "Generators/JsGenerator";
 import { Tokens } from "TokensGenerator";
 import { tsGenerator } from "Generators/TsGenerator";
+import { resolve } from "Utils/resolve";
 
 export type Generator = (
   tokens: Tokens,
@@ -15,3 +16,81 @@ export const Generators: Record<string, Generator> = {
   js: jsGenerator,
   ts: tsGenerator,
 };
+
+export const resolveRefs = (
+  tokens: Tokens,
+  callback: (ref: string, tokens: Tokens) => string
+): Tokens => {
+  let refsCount = 0;
+  let resolved = tokens;
+
+  // Each time we encounter a leaf with a value matching ref(...) we replacing it with the return value of callback(..).
+  const resolveRefsAux = (toResolve_: any) => {
+    if (typeof toResolve_ !== "object") {
+      const matches = /^ref\((.+)\)$/gm.exec(toResolve_);
+      if (!matches) {
+        return toResolve_;
+      }
+      refsCount++;
+      return callback(matches[1], resolved);
+    }
+    const resolvedSub: any = {};
+    Object.entries(toResolve_).forEach(([key, value]) => {
+      resolvedSub[key] = resolveRefsAux(value);
+    });
+    return resolvedSub;
+  };
+
+  // We need to resolveRefsAux until there is not refs found. In most cases there will be only two iterations
+  // ( one for resolving actual ref() and one another to make sure there were 0 ref() left ).
+  // But in some cases resolved ref() could result in a new ref(), so in the following example we will
+  // have three iterations:
+  //
+  // tokens:
+  // A = "foo"
+  // B = ref(A)
+  // C = ref(B)
+  //
+  // A = "foo"
+  // B = "foo"
+  // C = ref(A)
+  // it = 1
+  //
+  // A = "foo"
+  // B = "foo"
+  // C = "foo"
+  // it = 2
+  //
+  // A = "foo"
+  // B = "foo"
+  // C = "foo"
+  // it = 3
+  const maxIterations = 10;
+  let iterations = 0;
+  // eslint-disable-next-line no-constant-condition
+  do {
+    // Prevent infinite loops.
+    if (iterations >= maxIterations) {
+      throw new Error(
+        "Maximum resolveRefs iterations: please reduce usage of chained references."
+      );
+    }
+    refsCount = 0;
+    resolved = resolveRefsAux(resolved);
+    iterations++;
+  } while (refsCount > 0);
+
+  return resolved;
+};
+
+/**
+ * This function is meant to be given as callback to resolveRefs.
+ *
+ * It simply retrieves the actual value of "ref" ( dot notation like "theme.colors.primary-500" ) inside
+ * the resolvingTokens nested object.
+ * @param ref
+ * @param resolvingTokens
+ */
+export const resolveRefValue = (ref: string, resolvingTokens: Tokens) => {
+  return resolve(resolvingTokens, ref);
+};

packages/tokens/src/bin/ThemeGenerator.ts

@@ -11,7 +11,8 @@ export const buildTheme = async () => {
   const config = await getConfig();
   const tokens = tokensGenerator(config);
   const { generators } = options;
-  await Promise.allSettled(
+  // Promise.all() is used to propagates upward thrown errors.
+  await Promise.all(
     generators.map((generator: string) => {
       if (!Generators[generator]) {
         throw new Error('The generator "' + generator + '" does not exist.');

packages/tokens/src/bin/Utils/resolve.ts

@@ -0,0 +1,9 @@
+export const resolve = (
+  object: Record<string, any>,
+  path: string,
+  separator: string = "."
+): any => {
+  return path.split(separator).reduce((acc, cur) => {
+    return acc[cur];
+  }, object);
+};

packages/tokens/src/bin/__mocks__/cunningham.ts

@@ -23,5 +23,8 @@ module.exports = {
     transitions: {
       ease: "linear",
     },
+    input: {
+      "border-color": "ref(theme.colors.ternary-900)",
+    },
   },
 };

packages/tokens/src/bin/tests/assets/expected-default-cunningham-tokens.css

@@ -8,4 +8,5 @@
 	--c--theme--font--families--base: Roboto;
 	--c--theme--spacings--s: 1rem;
 	--c--theme--transitions--ease: linear;
+	--c--theme--input--border-color: var(--c--theme--colors--ternary-900);
 }

packages/tokens/src/bin/tests/assets/expected-js-cunningham-tokens.css

@@ -8,4 +8,5 @@
 	--c--theme--font--families--base: Roboto;
 	--c--theme--spacings--s: 1rem;
 	--c--theme--transitions--ease: linear;
+	--c--theme--input--border-color: var(--c--theme--colors--ternary-900);
 }

packages/tokens/src/bin/tests/assets/expected-with-utility-classes-cunningham-tokens.css

@@ -8,6 +8,7 @@
 	--c--theme--font--families--base: Roboto;
 	--c--theme--spacings--s: 1rem;
 	--c--theme--transitions--ease: linear;
+	--c--theme--input--border-color: var(--c--theme--colors--ternary-900);
 } .clr-primary { color: var(--c--theme--colors--primary); }
 .clr-secondary { color: var(--c--theme--colors--secondary); }
 .clr-ternary-900 { color: var(--c--theme--colors--ternary-900); }

packages/tokens/src/lib/cunningham-tokens.ts

@@ -1 +1 @@
-export const tokens = {"theme":{"colors":{"primary-text":"#FFFFFF","primary-100":"#EBF2FC","primary-200":"#8CB5EA","primary-300":"#5894E1","primary-400":"#377FDB","primary-500":"#055FD2","primary-600":"#0556BF","primary-700":"#044395","primary-800":"#033474","primary-900":"#022858","secondary-text":"#555F6B","secondary-100":"#F2F7FC","secondary-200":"#EBF3FA","secondary-300":"#E2EEF8","secondary-400":"#DDEAF7","secondary-500":"#D4E5F5","secondary-600":"#C1D0DF","secondary-700":"#97A3AE","secondary-800":"#757E87","secondary-900":"#596067","greyscale-000":"#FFFFFF","greyscale-100":"#FAFAFB","greyscale-200":"#F3F4F4","greyscale-300":"#E7E8EA","greyscale-400":"#C2C6CA","greyscale-500":"#9EA3AA","greyscale-600":"#79818A","greyscale-700":"#555F6B","greyscale-800":"#303C4B","greyscale-900":"#0C1A2B","success-text":"#FFFFFF","success-100":"#EFFCD3","success-200":"#DBFAA9","success-300":"#BEF27C","success-400":"#A0E659","success-500":"#76D628","success-600":"#5AB81D","success-700":"#419A14","success-800":"#2C7C0C","success-900":"#1D6607","info-text":"#FFFFFF","info-100":"#EBF2FC","info-200":"#8CB5EA","info-300":"#5894E1","info-400":"#377FDB","info-500":"#055FD2","info-600":"#0556BF","info-700":"#044395","info-800":"#033474","info-900":"#022858","warning-text":"#FFFFFF","warning-100":"#FFF8CD","warning-200":"#FFEF9B","warning-300":"#FFE469","warning-400":"#FFDA43","warning-500":"#FFC805","warning-600":"#DBA603","warning-700":"#B78702","warning-800":"#936901","warning-900":"#7A5400","danger-text":"#FFFFFF","danger-100":"#F4B0B0","danger-200":"#EE8A8A","danger-300":"#E65454","danger-400":"#E13333","danger-500":"#DA0000","danger-600":"#C60000","danger-700":"#9B0000","danger-800":"#780000","danger-900":"#5C0000"},"font":{"sizes":{"h1":"1.75rem","h2":"1.375rem","h3":"1.125rem","h4":"0.8125rem","h5":"0.625rem","h6":"0.5rem","l":"1rem","m":"0.8125rem","s":"0.6875rem"},"weights":{"thin":200,"light":300,"regular":400,"medium":500,"bold":600,"extrabold":700,"black":800},"families":{"base":"\"Roboto Flex Variable\", sans-serif","accent":"\"Roboto Flex Variable\", sans-serif"}},"spacings":{"xl":"4rem","l":"3rem","b":"1.625rem","s":"1rem","t":"0.5rem","st":"0.25rem"},"transitions":{"ease-in":"cubic-bezier(0.32, 0, 0.67, 0)","ease-out":"cubic-bezier(0.33, 1, 0.68, 1)","ease-in-out":"cubic-bezier(0.65, 0, 0.35, 1)","duration":"250ms"}}};
+export const tokens = {"theme":{"colors":{"primary-text":"#FFFFFF","primary-100":"#EBF2FC","primary-200":"#8CB5EA","primary-300":"#5894E1","primary-400":"#377FDB","primary-500":"#055FD2","primary-600":"#0556BF","primary-700":"#044395","primary-800":"#033474","primary-900":"#022858","secondary-text":"#555F6B","secondary-100":"#F2F7FC","secondary-200":"#EBF3FA","secondary-300":"#E2EEF8","secondary-400":"#DDEAF7","secondary-500":"#D4E5F5","secondary-600":"#C1D0DF","secondary-700":"#97A3AE","secondary-800":"#757E87","secondary-900":"#596067","greyscale-000":"#FFFFFF","greyscale-100":"#FAFAFB","greyscale-200":"#F3F4F4","greyscale-300":"#E7E8EA","greyscale-400":"#C2C6CA","greyscale-500":"#9EA3AA","greyscale-600":"#79818A","greyscale-700":"#555F6B","greyscale-800":"#303C4B","greyscale-900":"#0C1A2B","success-text":"#FFFFFF","success-100":"#EFFCD3","success-200":"#DBFAA9","success-300":"#BEF27C","success-400":"#A0E659","success-500":"#76D628","success-600":"#5AB81D","success-700":"#419A14","success-800":"#2C7C0C","success-900":"#1D6607","info-text":"#FFFFFF","info-100":"#EBF2FC","info-200":"#8CB5EA","info-300":"#5894E1","info-400":"#377FDB","info-500":"#055FD2","info-600":"#0556BF","info-700":"#044395","info-800":"#033474","info-900":"#022858","warning-text":"#FFFFFF","warning-100":"#FFF8CD","warning-200":"#FFEF9B","warning-300":"#FFE469","warning-400":"#FFDA43","warning-500":"#FFC805","warning-600":"#DBA603","warning-700":"#B78702","warning-800":"#936901","warning-900":"#7A5400","danger-text":"#FFFFFF","danger-100":"#F4B0B0","danger-200":"#EE8A8A","danger-300":"#E65454","danger-400":"#E13333","danger-500":"#DA0000","danger-600":"#C60000","danger-700":"#9B0000","danger-800":"#780000","danger-900":"#5C0000"},"font":{"sizes":{"h1":"1.75rem","h2":"1.375rem","h3":"1.125rem","h4":"0.8125rem","h5":"0.625rem","h6":"0.5rem","l":"1rem","m":"0.8125rem","s":"0.6875rem"},"weights":{"thin":{},"light":{},"regular":{},"medium":{},"bold":{},"extrabold":{},"black":{}},"families":{"base":"\"Roboto Flex Variable\", sans-serif","accent":"\"Roboto Flex Variable\", sans-serif"}},"spacings":{"xl":"4rem","l":"3rem","b":"1.625rem","s":"1rem","t":"0.5rem","st":"0.25rem"},"transitions":{"ease-in":"cubic-bezier(0.32, 0, 0.67, 0)","ease-out":"cubic-bezier(0.33, 1, 0.68, 1)","ease-in-out":"cubic-bezier(0.65, 0, 0.35, 1)","duration":"250ms"}}};

packages/tokens/src/lib/index.spec.ts

@@ -0,0 +1,21 @@
+import { buildRefs } from "./index";
+
+describe("buildRefs", () => {
+  it("should replace raw values by ref keys", () => {
+    expect(
+      buildRefs({
+        theme: {
+          colors: {
+            "primary-500": "blue",
+          },
+        },
+      })
+    ).toEqual({
+      theme: {
+        colors: {
+          "primary-500": "ref(theme.colors.primary-500)",
+        },
+      },
+    });
+  });
+});

packages/tokens/src/lib/index.ts

@@ -2,3 +2,39 @@ import { tokens } from "./cunningham-tokens";
 
 export type DefaultTokens = typeof tokens;
 export const defaultTokens = tokens;
+
+/**
+ * Transform such object:
+ * {
+ *     theme: {
+ *         colors: {
+ *            "primary-500": "blue"
+ *         }
+ *     }
+ * }
+ *
+ * to:
+ * {
+ *     theme: {
+ *         colors: {
+ *             "primary-500": "ref(theme.colors.primary-500)"
+ *         }
+ *     }
+ * }
+ * @param tokens_
+ */
+export const buildRefs = <T extends Object>(tokens_: T): T => {
+  const buildRefsAux = (upperKey: string, subTokens: any) => {
+    if (typeof subTokens === "object") {
+      const obj: any = {};
+      Object.entries(subTokens).forEach(([key, value]) => {
+        obj[key] = buildRefsAux((upperKey ? upperKey + "." : "") + key, value);
+      });
+      return obj;
+    }
+    return "ref(" + upperKey + ")";
+  };
+  return buildRefsAux("", tokens_);
+};
+
+export const defaultTokenRefs = buildRefs(defaultTokens);

packages/tokens/src/lib/tsconfig.json

@@ -8,4 +8,4 @@
     "outDir": "../../dist/lib",
     "declaration": true
   }
-}
+}