PocketFlow-Nim/examples/benchmarks.nim at main · bkataru/PocketFlow-Nim · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
## Performance Benchmarks
##
## Compares performance of different node types and configurations.

import asyncdispatch, json
import ../src/pocketflow

proc runBenchmarks() {.async.} =
  echo "=== PocketFlow Performance Benchmarks ===\n"

  let suite = newBenchmarkSuite()

  # Benchmark 1: Simple node execution
  let simpleNode = newNode(
    exec = proc(ctx: PfContext, params: JsonNode, prepRes: JsonNode): Future[JsonNode] {.async, closure, gcsafe.} =
      return %(42 * 2)
  )

  discard await suite.benchmarkNode("Simple Node", simpleNode, iterations = 1000)

  # Benchmark 2: Node with prep and post
  let complexNode = newNode(
    prep = proc(ctx: PfContext, params: JsonNode): Future[JsonNode] {.async, closure, gcsafe.} =
      return %*[1, 2, 3, 4, 5]
    ,
    exec = proc(ctx: PfContext, params: JsonNode, prepRes: JsonNode): Future[JsonNode] {.async, closure, gcsafe.} =
      var sum = 0
      for item in prepRes:
        sum += item.getInt()
      return %sum
    ,
    post = proc(ctx: PfContext, params: JsonNode, prepRes: JsonNode, execRes: JsonNode): Future[string] {.async, closure, gcsafe.} =
      return "done"
  )

  discard await suite.benchmarkNode("Complex Node (prep+exec+post)", complexNode, iterations = 1000)

  # Benchmark 3: BatchNode
  let batchNode = newBatchNode(
    prep = proc(ctx: PfContext, params: JsonNode): Future[JsonNode] {.async, closure, gcsafe.} =
      var items = newJArray()
      for i in 0..<10:
        items.add(%i)
      return items
    ,
    execItem = proc(ctx: PfContext, params: JsonNode, item: JsonNode): Future[JsonNode] {.async, closure, gcsafe.} =
      return %(item.getInt() * 2)
  )

  discard await suite.benchmarkNode("BatchNode (10 items)", batchNode, iterations = 100)

  # Benchmark 4: ParallelBatchNode (unlimited concurrency)
  let parallelBatchUnlimited = newParallelBatchNode(
    prep = proc(ctx: PfContext, params: JsonNode): Future[JsonNode] {.async, closure, gcsafe.} =
      var items = newJArray()
      for i in 0..<10:
        items.add(%i)
      return items
    ,
    execItem = proc(ctx: PfContext, params: JsonNode, item: JsonNode): Future[JsonNode] {.async, closure, gcsafe.} =
      await sleepAsync(1)  # Simulate work
      return %(item.getInt() * 2)
    ,
    maxConcurrency = 0
  )

  discard await suite.benchmarkNode("ParallelBatch (unlimited)", parallelBatchUnlimited, iterations = 50)

  # Benchmark 5: ParallelBatchNode (limited concurrency = 3)
  let parallelBatchLimited = newParallelBatchNode(
    prep = proc(ctx: PfContext, params: JsonNode): Future[JsonNode] {.async, closure, gcsafe.} =
      var items = newJArray()
      for i in 0..<10:
        items.add(%i)
      return items
    ,
    execItem = proc(ctx: PfContext, params: JsonNode, item: JsonNode): Future[JsonNode] {.async, closure, gcsafe.} =
      await sleepAsync(1)  # Simulate work
      return %(item.getInt() * 2)
    ,
    maxConcurrency = 3
  )

  discard await suite.benchmarkNode("ParallelBatch (limit=3)", parallelBatchLimited, iterations = 50)

  # Benchmark 6: Flow execution
  let node1 = newNode(
    exec = proc(ctx: PfContext, params: JsonNode, prepRes: JsonNode): Future[JsonNode] {.async, closure, gcsafe.} =
      return %10
  )
  let node2 = newNode(
    exec = proc(ctx: PfContext, params: JsonNode, prepRes: JsonNode): Future[JsonNode] {.async, closure, gcsafe.} =
      return %20
  )
  let node3 = newNode(
    exec = proc(ctx: PfContext, params: JsonNode, prepRes: JsonNode): Future[JsonNode] {.async, closure, gcsafe.} =
      return %30
  )

  let linearFlow = newFlow(node1 >> node2 >> node3)
  discard await suite.benchmarkFlow("Linear Flow (3 nodes)", linearFlow, iterations = 500)

  # Benchmark 7: ConditionalNode
  let conditionalNode = newConditionalNode(
    condition = proc(ctx: PfContext, params: JsonNode): Future[bool] {.async, closure, gcsafe.} =
      return true
    ,
    trueNode = newNode(
      exec = proc(ctx: PfContext, params: JsonNode, prepRes: JsonNode): Future[JsonNode] {.async, closure, gcsafe.} =
        return %"true_branch"
    ),
    falseNode = newNode(
      exec = proc(ctx: PfContext, params: JsonNode, prepRes: JsonNode): Future[JsonNode] {.async, closure, gcsafe.} =
        return %"false_branch"
    )
  )

  discard await suite.benchmarkNode("ConditionalNode", conditionalNode, iterations = 1000)

  # Benchmark 8: LoopNode
  let loopNode = newLoopNode(
    items = proc(ctx: PfContext, params: JsonNode): Future[JsonNode] {.async, closure, gcsafe.} =
      return %*[1, 2, 3, 4, 5]
    ,
    body = newNode(
      exec = proc(ctx: PfContext, params: JsonNode, prepRes: JsonNode): Future[JsonNode] {.async, closure, gcsafe.} =
        let item = ctx["__loop_item__"].getInt()
        ctx["__loop_result__"] = %(item * 2)
        return %(item * 2)
    )
  )

  discard await suite.benchmarkNode("LoopNode (5 iterations)", loopNode, iterations = 200)

  # Print results
  suite.printSummary()

  # Compare to baseline
  suite.compare("Simple Node")

  # Export to JSON
  echo "\nExporting results to benchmarks.json..."
  writeFile("benchmarks.json", suite.toJson().pretty())

when isMainModule:
  waitFor runBenchmarks()