Skip to content

Speeding up Tensor.tolist() #1191

@tarekziade

Description

@tarekziade

There might be a way to speed up Tensor.tolist() by simplifying its code

below is a side-by-side comparison of my version with the 3.0.2 one we ship in FF

var transformers = ChromeUtils.importESModule("chrome://global/content/ml/transformers-dev.js");

class CustomTensor extends transformers.Tensor {
    tolist() {
        const shape = this.dims;
        const data = this.data;
        if (shape.length === 1) return [...data];
        const result = new Array(shape[0]);
        const rowSize = shape[1];
        
        for (let i = 0; i < shape[0]; i++) {
            const row = new Array(rowSize);
            for (let j = 0; j < rowSize; j++) {
                row[j] = data[i * rowSize + j];
            }
            result[i] = row;
        }
        return result;
    }
}

async function test_tolist_speed(iterations = 10) {
    const numTokens = 16;
    const embeddingDim = 768;
    const totalElements = numTokens * embeddingDim;
    const data = Array.from({ length: totalElements }, () => Math.random() * 2 - 1);
    const shape = [numTokens, embeddingDim]; // [16, 768] for a sentence embedding
    
    const tensor = new transformers.Tensor('float32', data, shape);
    const customTensor = new CustomTensor('float32', data, shape);

    let totalTimeDefault = 0;
    let totalTimeOptimized = 0;
    let allEqual = true;

    function arraysEqual(arr1, arr2) {
        if (!Array.isArray(arr1) || !Array.isArray(arr2) || arr1.length !== arr2.length) return false;
        return arr1.every((val, index) => Array.isArray(val) ? arraysEqual(val, arr2[index]) : val === arr2[index]);
    }

    for (let i = 0; i < iterations; i++) {
        let start = performance.now();
        const defaultList = tensor.tolist();
        totalTimeDefault += performance.now() - start;
        
        start = performance.now();
        const optimizedList = customTensor.tolist();
        totalTimeOptimized += performance.now() - start;
        
        if (!arraysEqual(defaultList, optimizedList)) {
            allEqual = false;
        }
    }
    console.debug(`Avg tolist() time (default): ${(totalTimeDefault / iterations).toFixed(4)} ms`);
    console.debug(`Avg tolist() time (optimized): ${(totalTimeOptimized / iterations).toFixed(4)} ms`);
    console.debug(`Lists are equal: ${allEqual}`);
}


await test_tolist_speed(); 

And the results:

console.debug: "Avg tolist() time (default): 0.2163 ms"
console.debug: "Avg tolist() time (optimized): 0.0904 ms"
console.debug: "Lists are equal: true"

console.debug: "Avg tolist() time (default): 0.2083 ms"
console.debug: "Avg tolist() time (optimized): 0.0793 ms"
console.debug: "Lists are equal: true"

Let me know if that sounds correct or if I missed something - happy to contribute that patch

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type
    No fields configured for issues without a type.

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions