Add document transformation capability for indexing

extension.yaml

@@ -348,3 +348,22 @@ params:
         value: true
     default: false
     required: false
+  - param: TRANSFORM_FUNCTION_NAME
+    label: Transform Function Name
+    description: >-
+      Optional: Name of a Cloud Function to transform Firestore documents before indexing them into Typesense.
+      The function must be deployed in the same project and region as this extension.
+      If specified, this extension will call this function with each document before indexing.
+      Leave empty to skip transformation.
+    example: transformDoc
+    default: ""
+    required: false
+  - param: TRANSFORM_FUNCTION_SECRET
+    label: Transform Function Secret
+    type: secret
+    description: >-
+      Authorization secret for the transform function.
+      This will be sent in the Authorization header as "Bearer [secret]".
+      Only required if TRANSFORM_FUNCTION_NAME is set and the transform function requires authorization.
+    example: ""
+    required: false

functions/src/config.js

@@ -13,4 +13,8 @@ module.exports = {
   typesenseAPIKey: process.env.TYPESENSE_API_KEY,
   typesenseBackfillTriggerDocumentInFirestore: "typesense_sync/backfill",
   typesenseBackfillBatchSize: 1000,
+  transformFunctionName: process.env.TRANSFORM_FUNCTION_NAME,
+  transformFunctionSecret: process.env.TRANSFORM_FUNCTION_SECRET,
+  transformFunctionProjectId: process.env.GCLOUD_PROJECT,
+  transformFunctionRegion: process.env.LOCATION,
 };

functions/src/indexOnWrite.js

@@ -17,7 +17,13 @@ exports.indexOnWrite = onDocumentWritten(`${config.firestoreCollectionPath}/{doc
 
     // snapshot.after.ref.get() will refetch the latest version of the document
     const latestSnapshot = await snapshot.data.after.ref.get();
-    const typesenseDocument = await utils.typesenseDocumentFromSnapshot(latestSnapshot, snapshot.params);
+    let typesenseDocument;
+    if (config.transformFunctionName) {
+      const transformedDocument = await utils.transformDocument(latestSnapshot);
+      typesenseDocument = await utils.typesenseDocumentFromSnapshot(transformedDocument, snapshot.params);
+    } else {
+      typesenseDocument = await utils.typesenseDocumentFromSnapshot(latestSnapshot, snapshot.params);
+    }
 
     if (config.shouldLogTypesenseInserts) {
       debug(`Upserting document ${JSON.stringify(typesenseDocument)}`);

functions/src/utils.js

@@ -1,5 +1,5 @@
 const config = require("./config.js");
-
+const {error, debug} = require("firebase-functions/logger");
 const mapValue = (value) => {
   const isObject = typeof value === "object";
   const notNull = value !== null;
@@ -223,3 +223,50 @@ exports.pathMatchesSelector = function (path, selector) {
 
   return extractedValues;
 };
+
+/**
+ * Transforms a document by calling a user-defined transform function
+ * If no transform function is defined, returns the original document
+ * @param {Object} document - The document to transform
+ * @return {Object} The transformed document
+ */
+exports.transformDocument = async function (document) {
+  const transformFunctionName = config.transformFunctionName;
+
+  if (!transformFunctionName) {
+    error("No transform function defined. Returning original document.");
+    return document;
+  }
+
+  try {
+    const projectId = config.transformFunctionProjectId;
+    const region = config.transformFunctionRegion;
+
+    debug(`Calling transform function: ${transformFunctionName}`);
+
+    const url = `https://${region}-${projectId}.cloudfunctions.net/${transformFunctionName}`;
+
+    const response = await fetch(url, {
+      method: "POST",
+      body: JSON.stringify({document}),
+      headers: {"Content-Type": "application/json", Authorization: `Bearer ${config.transformFunctionSecret}`},
+    });
+
+    if (!response.ok) {
+      throw new Error(`Error calling transform function: ${response.statusText}`);
+    }
+
+    const responseData = await response.json();
+
+    if (responseData) {
+      debug(`Transform function succeeded for document ${document.id || "unknown"}`);
+      return responseData;
+    }
+
+    debug(`Transform function failed for document ${document.id || "unknown"}. Using original document.`);
+    return document;
+  } catch (err) {
+    error(`Error calling transform function: ${err.message}`);
+    return document;
+  }
+};

package.json

@@ -4,10 +4,11 @@
   "scripts": {
     "emulator": "cross-env DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:start --import=emulator_data",
     "export": "firebase emulators:export emulator_data",
-    "test": "npm run test:flatttened && npm run test:unflattened && npm run test:subcollection",
+    "test": "npm run test:flatttened && npm run test:unflattened && npm run test:subcollection && npm run test:utils",
     "test:flatttened": "cp -f extensions/test-params-flatten-nested-true.local.env functions/.env && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions  'jest --testRegex=\"WithFlattening\" --testRegex=\"backfill.spec\"'",
     "test:unflattened": "cp -f extensions/test-params-flatten-nested-false.local.env functions/.env && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithoutFlattening\"'",
     "test:subcollection": "jest --testRegex=\"writeLogging\" --testRegex=\"Subcollection\" --detectOpenHandles",
+    "test:utils": "jest --testRegex=\"utilsTransform\"",
     "typesenseServer": "docker compose up",
     "lint:fix": "eslint . --fix",
     "lint": "eslint .",

test/utilsTransform.spec.js

@@ -0,0 +1,136 @@
+const {TestEnvironment} = require("./support/testEnvironment");
+
+const mockFetch = jest.fn();
+global.fetch = (...args) => mockFetch(...args);
+
+describe("Utils - transformDocument", () => {
+  let testEnvironment;
+  let utils;
+  let config;
+
+  beforeAll((done) => {
+    testEnvironment = new TestEnvironment({
+      dotenvConfig: `
+LOCATION=us-central1
+FIRESTORE_DATABASE_REGION=nam5
+FIRESTORE_COLLECTION_PATH=books
+FIRESTORE_COLLECTION_FIELDS=author,title
+TYPESENSE_HOSTS=localhost
+TYPESENSE_PORT=8108
+TYPESENSE_PROTOCOL=http
+TYPESENSE_COLLECTION_NAME=books_firestore
+TYPESENSE_API_KEY=xyz
+TRANSFORM_FUNCTION_NAME=
+TRANSFORM_FUNCTION_SECRET=test-secret
+GCLOUD_PROJECT=test-project
+`,
+    });
+
+    testEnvironment.setupTestEnvironment(done);
+  });
+
+  beforeEach(() => {
+    utils = require("../functions/src/utils.js");
+    config = require("../functions/src/config.js");
+
+    mockFetch.mockReset();
+
+    testEnvironment.resetCapturedEmulatorLogs();
+  });
+
+  afterAll(async () => {
+    await testEnvironment.teardownTestEnvironment();
+  });
+
+  describe("when no transform function is defined", () => {
+    it("returns the original document and logs an error", async () => {
+      config.transformFunctionName = null;
+
+      const document = {id: "123", title: "Test Document"};
+      const result = await utils.transformDocument(document);
+
+      expect(result).toEqual(document);
+
+      expect(mockFetch).not.toHaveBeenCalled();
+    });
+  });
+
+  describe("when transform function is defined", () => {
+    beforeEach(() => {
+      config.transformFunctionName = "test-transform-function";
+    });
+
+    it("successfully transforms a document", async () => {
+      const document = {id: "123", title: "Test Document"};
+      const transformedDocument = {id: "123", title: "Transformed Document"};
+
+      mockFetch.mockResolvedValueOnce({
+        ok: true,
+        json: async () => transformedDocument,
+      });
+
+      const result = await utils.transformDocument(document);
+
+      expect(result).toEqual(transformedDocument);
+
+      expect(mockFetch).toHaveBeenCalledWith("https://us-central1-test-project.cloudfunctions.net/test-transform-function", {
+        method: "POST",
+        body: JSON.stringify({document}),
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: "Bearer test-secret",
+        },
+      });
+    });
+
+    it("returns original document when transform function returns null/undefined", async () => {
+      const document = {id: "123", title: "Test Document"};
+
+      mockFetch.mockResolvedValueOnce({
+        ok: true,
+        json: async () => null,
+      });
+
+      const result = await utils.transformDocument(document);
+
+      expect(result).toEqual(document);
+    });
+
+    it("returns original document when fetch response is not OK", async () => {
+      const document = {id: "123", title: "Test Document"};
+
+      mockFetch.mockResolvedValueOnce({
+        ok: false,
+        statusText: "Internal Server Error",
+      });
+
+      const result = await utils.transformDocument(document);
+
+      expect(result).toEqual(document);
+    });
+
+    it("returns original document when fetch throws an exception", async () => {
+      const document = {id: "123", title: "Test Document"};
+
+      mockFetch.mockRejectedValueOnce(new Error("Network error"));
+
+      const result = await utils.transformDocument(document);
+
+      expect(result).toEqual(document);
+    });
+
+    it("handles documents without an ID properly", async () => {
+      const document = {title: "Test Document Without ID"};
+      const transformedDocument = {title: "Transformed Document"};
+
+      mockFetch.mockResolvedValueOnce({
+        ok: true,
+        json: async () => transformedDocument,
+      });
+
+      const result = await utils.transformDocument(document);
+
+      expect(result).toEqual(transformedDocument);
+    });
+  });
+});