From 98c81f27917172eae3d6651fed4486572b27cb37 Mon Sep 17 00:00:00 2001 From: Fanis Tharropoulos Date: Mon, 14 Apr 2025 15:10:05 +0300 Subject: [PATCH 1/9] feat: add transform func config params --- functions/src/config.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/functions/src/config.js b/functions/src/config.js index 71a7b5f..18e16d9 100644 --- a/functions/src/config.js +++ b/functions/src/config.js @@ -13,4 +13,8 @@ module.exports = { typesenseAPIKey: process.env.TYPESENSE_API_KEY, typesenseBackfillTriggerDocumentInFirestore: "typesense_sync/backfill", typesenseBackfillBatchSize: 1000, + transformFunctionName: process.env.TRANSFORM_FUNCTION_NAME, + transformFunctionSecret: process.env.TRANSFORM_FUNCTION_SECRET, + transformFunctionProjectId: process.env.TRANSFORM_FUNCTION_PROJECT_ID || process.env.GCLOUD_PROJECT, + transformFunctionRegion: process.env.TRANSFORM_FUNCTION_REGION || process.env.REGION, }; From eb41fea2534549639dbb2673c67baae7103991e4 Mon Sep 17 00:00:00 2001 From: Fanis Tharropoulos Date: Mon, 14 Apr 2025 15:11:40 +0300 Subject: [PATCH 2/9] feat(transform): add document transformation capability - add `transformDocument` function to call external transformation functions - implement error handling for failed transformations with fallback to original doc - add comprehensive tests for the transformation functionality - update package.json with new test command --- functions/src/utils.js | 49 +++++++++++- package.json | 1 + test/utilsTransform.spec.js | 149 ++++++++++++++++++++++++++++++++++++ 3 files changed, 198 insertions(+), 1 deletion(-) create mode 100644 test/utilsTransform.spec.js diff --git a/functions/src/utils.js b/functions/src/utils.js index f86a1fd..afe2bba 100644 --- a/functions/src/utils.js +++ b/functions/src/utils.js @@ -1,5 +1,5 @@ const config = require("./config.js"); - +const {error, debug} = require("firebase-functions/logger"); const mapValue = (value) => { const isObject = typeof value === "object"; const notNull = value !== null; @@ -223,3 +223,50 @@ exports.pathMatchesSelector = function (path, selector) { return extractedValues; }; + +/** + * Transforms a document by calling a user-defined transform function + * If no transform function is defined, returns the original document + * @param {Object} document - The document to transform + * @return {Object} The transformed document + */ +exports.transformDocument = async function (document) { + const transformFunctionName = config.transformFunctionName; + + if (!transformFunctionName) { + error("No transform function defined. Returning original document."); + return document; + } + + try { + const projectId = config.transformFunctionProjectId; + const region = config.transformFunctionRegion; + + debug(`Calling transform function: ${transformFunctionName}`); + + const url = `https://${region}-${projectId}.cloudfunctions.net/${transformFunctionName}`; + + const response = await fetch(url, { + method: "POST", + body: JSON.stringify({document}), + headers: {"Content-Type": "application/json", Authorization: `Bearer ${config.transformFunctionSecret}`}, + }); + + if (!response.ok) { + throw new Error(`Error calling transform function: ${response.statusText}`); + } + + const responseData = await response.json(); + + if (responseData) { + debug(`Transform function succeeded for document ${document.id || "unknown"}`); + return responseData; + } + + debug(`Transform function failed for document ${document.id || "unknown"}. Using original document.`); + return document; + } catch (err) { + error(`Error calling transform function: ${err.message}`); + return document; + } +}; diff --git a/package.json b/package.json index ee23fe8..c966a8a 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "test:flatttened": "cp -f extensions/test-params-flatten-nested-true.local.env functions/.env && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithFlattening\" --testRegex=\"backfill.spec\"'", "test:unflattened": "cp -f extensions/test-params-flatten-nested-false.local.env functions/.env && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithoutFlattening\"'", "test:subcollection": "jest --testRegex=\"writeLogging\" --testRegex=\"Subcollection\" --detectOpenHandles", + "test:utils": "jest --testRegex=\"utilsTransform\"", "typesenseServer": "docker compose up", "lint:fix": "eslint . --fix", "lint": "eslint .", diff --git a/test/utilsTransform.spec.js b/test/utilsTransform.spec.js new file mode 100644 index 0000000..84f5ade --- /dev/null +++ b/test/utilsTransform.spec.js @@ -0,0 +1,149 @@ +const {TestEnvironment} = require("./support/testEnvironment"); + +const mockFetch = jest.fn(); +global.fetch = (...args) => mockFetch(...args); + +describe("Utils - transformDocument", () => { + let testEnvironment; + let utils; + let config; + + beforeAll((done) => { + testEnvironment = new TestEnvironment({ + dotenvConfig: ` +LOCATION=us-central1 +FIRESTORE_DATABASE_REGION=nam5 +FIRESTORE_COLLECTION_PATH=books +FIRESTORE_COLLECTION_FIELDS=author,title +TYPESENSE_HOSTS=localhost +TYPESENSE_PORT=8108 +TYPESENSE_PROTOCOL=http +TYPESENSE_COLLECTION_NAME=books_firestore +TYPESENSE_API_KEY=xyz +TRANSFORM_FUNCTION_NAME= +TRANSFORM_FUNCTION_SECRET=test-secret +TRANSFORM_FUNCTION_PROJECT_ID=test-project +TRANSFORM_FUNCTION_REGION=us-central1 +`, + }); + + testEnvironment.setupTestEnvironment(done); + }); + + beforeEach(() => { + utils = require("../functions/src/utils.js"); + config = require("../functions/src/config.js"); + + mockFetch.mockReset(); + + testEnvironment.resetCapturedEmulatorLogs(); + }); + + afterAll(async () => { + await testEnvironment.teardownTestEnvironment(); + }); + + describe("when no transform function is defined", () => { + it("returns the original document and logs an error", async () => { + config.transformFunctionName = null; + + const document = {id: "123", title: "Test Document"}; + const result = await utils.transformDocument(document); + + expect(result).toEqual(document); + + expect(mockFetch).not.toHaveBeenCalled(); + + expect(testEnvironment.capturedEmulatorLogs).toContain("No transform function defined. Returning original document."); + }); + }); + + describe("when transform function is defined", () => { + beforeEach(() => { + config.transformFunctionName = "test-transform-function"; + }); + + it("successfully transforms a document", async () => { + const document = {id: "123", title: "Test Document"}; + const transformedDocument = {id: "123", title: "Transformed Document"}; + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => transformedDocument, + }); + + const result = await utils.transformDocument(document); + + expect(result).toEqual(transformedDocument); + + expect(mockFetch).toHaveBeenCalledWith("https://us-central1-test-project.cloudfunctions.net/test-transform-function", { + method: "POST", + body: JSON.stringify({document}), + headers: { + "Content-Type": "application/json", + Authorization: "Bearer test-secret", + }, + }); + + expect(testEnvironment.capturedEmulatorLogs).toContain("Calling transform function: test-transform-function"); + expect(testEnvironment.capturedEmulatorLogs).toContain("Transform function succeeded for document 123"); + }); + + it("returns original document when transform function returns null/undefined", async () => { + const document = {id: "123", title: "Test Document"}; + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => null, + }); + + const result = await utils.transformDocument(document); + + expect(result).toEqual(document); + + expect(testEnvironment.capturedEmulatorLogs).toContain("Transform function failed for document 123. Using original document."); + }); + + it("returns original document when fetch response is not OK", async () => { + const document = {id: "123", title: "Test Document"}; + + mockFetch.mockResolvedValueOnce({ + ok: false, + statusText: "Internal Server Error", + }); + + const result = await utils.transformDocument(document); + + expect(result).toEqual(document); + + }); + + it("returns original document when fetch throws an exception", async () => { + const document = {id: "123", title: "Test Document"}; + + mockFetch.mockRejectedValueOnce(new Error("Network error")); + + const result = await utils.transformDocument(document); + + expect(result).toEqual(document); + + expect(testEnvironment.capturedEmulatorLogs).toContain("Error calling transform function: Network error"); + }); + + it("handles documents without an ID properly", async () => { + const document = {title: "Test Document Without ID"}; + const transformedDocument = {title: "Transformed Document"}; + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => transformedDocument, + }); + + const result = await utils.transformDocument(document); + + expect(result).toEqual(transformedDocument); + + expect(testEnvironment.capturedEmulatorLogs).toContain("Transform function succeeded for document unknown"); + }); + }); +}); From 958ccf1a756732cc8d2248f27149390873ef7237 Mon Sep 17 00:00:00 2001 From: Fanis Tharropoulos Date: Mon, 14 Apr 2025 15:12:24 +0300 Subject: [PATCH 3/9] feat(indexing): integrate document transformation in indexOnWrite - update indexOnWrite to conditionally use transformation function - check for transformFunctionName in config before transforming - apply transformation before converting to typesense document --- functions/src/indexOnWrite.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/functions/src/indexOnWrite.js b/functions/src/indexOnWrite.js index c3fded7..7446989 100644 --- a/functions/src/indexOnWrite.js +++ b/functions/src/indexOnWrite.js @@ -17,7 +17,13 @@ exports.indexOnWrite = onDocumentWritten(`${config.firestoreCollectionPath}/{doc // snapshot.after.ref.get() will refetch the latest version of the document const latestSnapshot = await snapshot.data.after.ref.get(); - const typesenseDocument = await utils.typesenseDocumentFromSnapshot(latestSnapshot, snapshot.params); + let typesenseDocument; + if (config.transformFunctionName) { + const transformedDocument = await utils.transformDocument(latestSnapshot); + typesenseDocument = await utils.typesenseDocumentFromSnapshot(transformedDocument, snapshot.params); + } else { + typesenseDocument = await utils.typesenseDocumentFromSnapshot(latestSnapshot, snapshot.params); + } if (config.shouldLogTypesenseInserts) { debug(`Upserting document ${JSON.stringify(typesenseDocument)}`); From 3b2c12e77fb547faa545ee770bd04e7e4aee41d2 Mon Sep 17 00:00:00 2001 From: Fanis Tharropoulos Date: Mon, 14 Apr 2025 15:12:43 +0300 Subject: [PATCH 4/9] feat(config): add document transformation configuration options - add TRANSFORM_FUNCTION_NAME parameter for specifying transform function - add TRANSFORM_FUNCTION_PROJECT_ID for cross-project function support - add TRANSFORM_FUNCTION_REGION for region specification - add TRANSFORM_FUNCTION_SECRET for authorization to protected functions --- extension.yaml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/extension.yaml b/extension.yaml index 7cdaad6..20e6ec0 100644 --- a/extension.yaml +++ b/extension.yaml @@ -348,3 +348,39 @@ params: value: true default: false required: false + - param: TRANSFORM_FUNCTION_NAME + label: Transform Function Name + description: >- + Optional: Name of a Cloud Function to transform Firestore documents before indexing them into Typesense. + If specified, this extension will call this function with each document before indexing. + Leave empty to skip transformation. + example: transformDoc + default: "" + required: false + - param: TRANSFORM_FUNCTION_PROJECT_ID + label: Transform Function Project ID + description: >- + Project ID where the transform function is deployed. + Only required if TRANSFORM_FUNCTION_NAME is set. + Defaults to the current project ID if not specified. + example: my-project + default: "" + required: false + - param: TRANSFORM_FUNCTION_REGION + label: Transform Function Region + description: >- + Region where the transform function is deployed. + Only required if TRANSFORM_FUNCTION_NAME is set. + Defaults to the same region as this extension if not specified. + example: us-central1 + default: "" + required: false + - param: TRANSFORM_FUNCTION_SECRET + label: Transform Function Secret + type: secret + description: >- + Authorization secret for the transform function. + This will be sent in the Authorization header as "Bearer [secret]". + Only required if TRANSFORM_FUNCTION_NAME is set and the transform function requires authorization. + example: "" + required: false From 55ebed99f5b82c8572e0816c84a1707afb245ab3 Mon Sep 17 00:00:00 2001 From: Fanis Tharropoulos Date: Mon, 14 Apr 2025 15:14:31 +0300 Subject: [PATCH 5/9] fix: run util tests on default test script --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c966a8a..5c450b1 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "scripts": { "emulator": "cross-env DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:start --import=emulator_data", "export": "firebase emulators:export emulator_data", - "test": "npm run test:flatttened && npm run test:unflattened && npm run test:subcollection", + "test": "npm run test:flatttened && npm run test:unflattened && npm run test:subcollection && npm run test:utils", "test:flatttened": "cp -f extensions/test-params-flatten-nested-true.local.env functions/.env && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithFlattening\" --testRegex=\"backfill.spec\"'", "test:unflattened": "cp -f extensions/test-params-flatten-nested-false.local.env functions/.env && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithoutFlattening\"'", "test:subcollection": "jest --testRegex=\"writeLogging\" --testRegex=\"Subcollection\" --detectOpenHandles", From 906db87c94a44f17a1787301a5f7220c32873861 Mon Sep 17 00:00:00 2001 From: Fanis Tharropoulos Date: Mon, 14 Apr 2025 15:23:10 +0300 Subject: [PATCH 6/9] chore: lint --- test/utilsTransform.spec.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/utilsTransform.spec.js b/test/utilsTransform.spec.js index 84f5ade..592768f 100644 --- a/test/utilsTransform.spec.js +++ b/test/utilsTransform.spec.js @@ -115,7 +115,6 @@ TRANSFORM_FUNCTION_REGION=us-central1 const result = await utils.transformDocument(document); expect(result).toEqual(document); - }); it("returns original document when fetch throws an exception", async () => { From f89ea4193df9aadbffc6e5eaea33de46a2fdf697 Mon Sep 17 00:00:00 2001 From: Fanis Tharropoulos Date: Mon, 14 Apr 2025 15:32:36 +0300 Subject: [PATCH 7/9] chore: remove logging checks --- test/utilsTransform.spec.js | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/test/utilsTransform.spec.js b/test/utilsTransform.spec.js index 592768f..591066d 100644 --- a/test/utilsTransform.spec.js +++ b/test/utilsTransform.spec.js @@ -53,8 +53,6 @@ TRANSFORM_FUNCTION_REGION=us-central1 expect(result).toEqual(document); expect(mockFetch).not.toHaveBeenCalled(); - - expect(testEnvironment.capturedEmulatorLogs).toContain("No transform function defined. Returning original document."); }); }); @@ -84,9 +82,6 @@ TRANSFORM_FUNCTION_REGION=us-central1 Authorization: "Bearer test-secret", }, }); - - expect(testEnvironment.capturedEmulatorLogs).toContain("Calling transform function: test-transform-function"); - expect(testEnvironment.capturedEmulatorLogs).toContain("Transform function succeeded for document 123"); }); it("returns original document when transform function returns null/undefined", async () => { @@ -100,8 +95,6 @@ TRANSFORM_FUNCTION_REGION=us-central1 const result = await utils.transformDocument(document); expect(result).toEqual(document); - - expect(testEnvironment.capturedEmulatorLogs).toContain("Transform function failed for document 123. Using original document."); }); it("returns original document when fetch response is not OK", async () => { @@ -125,8 +118,6 @@ TRANSFORM_FUNCTION_REGION=us-central1 const result = await utils.transformDocument(document); expect(result).toEqual(document); - - expect(testEnvironment.capturedEmulatorLogs).toContain("Error calling transform function: Network error"); }); it("handles documents without an ID properly", async () => { @@ -141,8 +132,6 @@ TRANSFORM_FUNCTION_REGION=us-central1 const result = await utils.transformDocument(document); expect(result).toEqual(transformedDocument); - - expect(testEnvironment.capturedEmulatorLogs).toContain("Transform function succeeded for document unknown"); }); }); }); From f5c6b8ec886e8dccbffaab4deab0d60464238f0e Mon Sep 17 00:00:00 2001 From: Fanis Tharropoulos Date: Wed, 18 Jun 2025 14:26:10 +0300 Subject: [PATCH 8/9] refactor(config): simplify transform function configuration defaults - Remove `TRANSFORM_FUNCTION_PROJECT_ID` and `TRANSFORM_FUNCTION_REGION` parameters from `extension.yaml` - Update `config.js` to use `GCLOUD_PROJECT` and `LOCATION` environment --- extension.yaml | 19 +------------------ functions/src/config.js | 4 ++-- test/utilsTransform.spec.js | 2 -- 3 files changed, 3 insertions(+), 22 deletions(-) diff --git a/extension.yaml b/extension.yaml index 20e6ec0..79bcb35 100644 --- a/extension.yaml +++ b/extension.yaml @@ -352,29 +352,12 @@ params: label: Transform Function Name description: >- Optional: Name of a Cloud Function to transform Firestore documents before indexing them into Typesense. + The function must be deployed in the same project and region as this extension. If specified, this extension will call this function with each document before indexing. Leave empty to skip transformation. example: transformDoc default: "" required: false - - param: TRANSFORM_FUNCTION_PROJECT_ID - label: Transform Function Project ID - description: >- - Project ID where the transform function is deployed. - Only required if TRANSFORM_FUNCTION_NAME is set. - Defaults to the current project ID if not specified. - example: my-project - default: "" - required: false - - param: TRANSFORM_FUNCTION_REGION - label: Transform Function Region - description: >- - Region where the transform function is deployed. - Only required if TRANSFORM_FUNCTION_NAME is set. - Defaults to the same region as this extension if not specified. - example: us-central1 - default: "" - required: false - param: TRANSFORM_FUNCTION_SECRET label: Transform Function Secret type: secret diff --git a/functions/src/config.js b/functions/src/config.js index 18e16d9..8c10957 100644 --- a/functions/src/config.js +++ b/functions/src/config.js @@ -15,6 +15,6 @@ module.exports = { typesenseBackfillBatchSize: 1000, transformFunctionName: process.env.TRANSFORM_FUNCTION_NAME, transformFunctionSecret: process.env.TRANSFORM_FUNCTION_SECRET, - transformFunctionProjectId: process.env.TRANSFORM_FUNCTION_PROJECT_ID || process.env.GCLOUD_PROJECT, - transformFunctionRegion: process.env.TRANSFORM_FUNCTION_REGION || process.env.REGION, + transformFunctionProjectId: process.env.GCLOUD_PROJECT, + transformFunctionRegion: process.env.LOCATION, }; diff --git a/test/utilsTransform.spec.js b/test/utilsTransform.spec.js index 591066d..90712b2 100644 --- a/test/utilsTransform.spec.js +++ b/test/utilsTransform.spec.js @@ -22,8 +22,6 @@ TYPESENSE_COLLECTION_NAME=books_firestore TYPESENSE_API_KEY=xyz TRANSFORM_FUNCTION_NAME= TRANSFORM_FUNCTION_SECRET=test-secret -TRANSFORM_FUNCTION_PROJECT_ID=test-project -TRANSFORM_FUNCTION_REGION=us-central1 `, }); From bbb32a4a20684e525a45038a24f0a1ccb5185146 Mon Sep 17 00:00:00 2001 From: Fanis Tharropoulos Date: Wed, 18 Jun 2025 15:14:15 +0300 Subject: [PATCH 9/9] fix(test): fix env var in transform test --- test/utilsTransform.spec.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/utilsTransform.spec.js b/test/utilsTransform.spec.js index 90712b2..c40d68a 100644 --- a/test/utilsTransform.spec.js +++ b/test/utilsTransform.spec.js @@ -22,6 +22,7 @@ TYPESENSE_COLLECTION_NAME=books_firestore TYPESENSE_API_KEY=xyz TRANSFORM_FUNCTION_NAME= TRANSFORM_FUNCTION_SECRET=test-secret +GCLOUD_PROJECT=test-project `, });