Add ability to make the FE trigger automatic dashboard refresh (#6442)

* implement automatic reload

* single source of truth: pass meta tag from the BE

* improve naming

* add test

* fix FE tests

- extract fetchSuggestions into a separate module to break circular dependency
- handle headers not existing in API response (no headers in tests)

* increase version to test on staging

* works, revert version increase

* test x-api-version appearing in /query response headers

* console log -> warn

* move condition to maybeReload fn

* reload only for idempotent requests

* return effective api_version as minimum across the cluster

* FE: reload only when effective version is greater than expected + tests

* parse and stringify to add search param

* ETS approach + default to 0 before first multicall response

* fallback to 0 in effective_version instead

* remove redundant start_link in test_helper

* test increase version

* revert increase

Author: RobertJoonas

assets/js/dashboard/api.ts

@@ -7,6 +7,7 @@ import { serializeApiFilters } from './util/filters'
 import * as url from './util/url'
 import { MainGraphResponse } from './stats/graph/fetch-main-graph'
 import { CsvExportRequestBody } from './stats/csv-export/csv-export-body'
+import { maybeReloadForApiVersion } from './util/url-search-params'
 
 let abortController = new AbortController()
 let SHARED_LINK_AUTH: null | string = null
@@ -148,7 +149,14 @@ async function throwApiErrorIfNotOk(response: Response) {
   }
 }
 
-async function handleApiResponse(response: Response) {
+async function handleApiResponse(
+  response: Response,
+  opts: Record<'idempotent', boolean> = { idempotent: true }
+) {
+  if (opts.idempotent) {
+    maybeReloadForApiVersion(window.location, response.headers)
+  }
+
   await throwApiErrorIfNotOk(response)
   return response.json()
 }
@@ -262,5 +270,5 @@ export const mutation = async <
     body: fetchOptions.body,
     signal: abortController.signal
   })
-  return handleApiResponse(response)
+  return handleApiResponse(response, { idempotent: false })
 }

assets/js/dashboard/stats/modals/filter-modal-props-row.js

@@ -8,10 +8,10 @@ import { apiPath } from '../../util/url'
 import {
   EVENT_PROPS_PREFIX,
   FILTER_OPERATIONS,
-  fetchSuggestions,
   getPropertyKeyFromFilterKey,
   isFreeChoiceFilterOperation
 } from '../../util/filters'
+import { fetchSuggestions } from '../../util/fetch-suggestions'
 import { useDashboardStateContext } from '../../dashboard-state-context'
 import { useSiteContext } from '../../site-context'
 

assets/js/dashboard/stats/modals/filter-modal-row.js

@@ -7,11 +7,11 @@ import Combobox from '../../components/combobox'
 
 import {
   FILTER_OPERATIONS,
-  fetchSuggestions,
   isFreeChoiceFilterOperation,
   getLabel,
   formattedFilters
 } from '../../util/filters'
+import { fetchSuggestions } from '../../util/fetch-suggestions'
 import { apiPath } from '../../util/url'
 import { useDashboardStateContext } from '../../dashboard-state-context'
 import { useSiteContext } from '../../site-context'

assets/js/dashboard/util/fetch-suggestions.ts

@@ -0,0 +1,35 @@
+import * as api from '../api'
+import { DashboardState, Filter } from '../dashboard-state'
+import { replaceFilterByPrefix, omitFiltersByKeyPrefix } from './filters'
+
+export function fetchSuggestions(
+  apiPath: string,
+  dashboardState: DashboardState,
+  input: string,
+  additionalFilter?: Filter
+) {
+  const updatedQuery = queryForSuggestions(dashboardState, additionalFilter)
+  return api.get(apiPath, updatedQuery, { q: input.trim() })
+}
+
+function queryForSuggestions(
+  dashboardState: DashboardState,
+  additionalFilter?: Filter
+): DashboardState {
+  let filters = dashboardState.filters
+  if (additionalFilter) {
+    const [_operation, filterKey, clauses] = additionalFilter
+
+    // For suggestions, we remove already-applied filter with same key from dashboardState and add new filter (if feasible)
+    if (clauses.length > 0) {
+      filters = replaceFilterByPrefix(
+        dashboardState,
+        filterKey,
+        additionalFilter
+      )
+    } else {
+      filters = omitFiltersByKeyPrefix(dashboardState, filterKey)
+    }
+  }
+  return { ...dashboardState, filters }
+}

assets/js/dashboard/util/filters.js

@@ -1,4 +1,3 @@
-import * as api from '../api'
 import { formatSegmentIdAsLabelKey } from '../filtering/segments'
 
 export const FILTER_MODAL_TO_FILTER_GROUP = {
@@ -95,7 +94,7 @@ const hasDimensionPrefix =
   ([_operation, dimension, _clauses]) =>
     dimension.startsWith(prefix)
 
-function omitFiltersByKeyPrefix(dashboardState, prefix) {
+export function omitFiltersByKeyPrefix(dashboardState, prefix) {
   return dashboardState.filters.filter(
     ([_operation, filterKey, _clauses]) => !filterKey.startsWith(prefix)
   )
@@ -279,35 +278,6 @@ function remapToApiFilter([operation, filterKey, clauses, ...modifiers]) {
   }
 }
 
-export function fetchSuggestions(
-  apiPath,
-  dashboardState,
-  input,
-  additionalFilter
-) {
-  const updatedQuery = queryForSuggestions(dashboardState, additionalFilter)
-  return api.get(apiPath, updatedQuery, { q: input.trim() })
-}
-
-function queryForSuggestions(dashboardState, additionalFilter) {
-  let filters = dashboardState.filters
-  if (additionalFilter) {
-    const [_operation, filterKey, clauses] = additionalFilter
-
-    // For suggestions, we remove already-applied filter with same key from dashboardState and add new filter (if feasible)
-    if (clauses.length > 0) {
-      filters = replaceFilterByPrefix(
-        dashboardState,
-        filterKey,
-        additionalFilter
-      )
-    } else {
-      filters = omitFiltersByKeyPrefix(dashboardState, filterKey)
-    }
-  }
-  return { ...dashboardState, filters }
-}
-
 export function getFilterGroup([_operation, filterKey, _clauses]) {
   return filterKey.startsWith(EVENT_PROPS_PREFIX) ? 'props' : filterKey
 }

assets/js/dashboard/util/url-search-params.test.ts

@@ -4,6 +4,7 @@ import {
   getSearchWithEnforcedSegment,
   isSearchEntryDefined,
   maybeGetLatestReadableSearch,
+  maybeReloadForApiVersion,
   parseFilter,
   parseLabelsEntry,
   parseSearch,
@@ -259,3 +260,64 @@ describe(`${getSearchWithEnforcedSegment.name}`, () => {
     ).toEqual(expectedUpdatedSearch)
   })
 })
+
+describe(`${maybeReloadForApiVersion.name}`, () => {
+  const dashboardPathname = '/example.com'
+
+  type MockWindowLocation = Location & { replace: jest.Mock }
+
+  function makeLocation(search: string): MockWindowLocation {
+    return {
+      pathname: dashboardPathname,
+      search,
+      hash: '',
+      replace: jest.fn()
+    } as unknown as MockWindowLocation
+  }
+
+  function makeHeaders(version: string | null): Headers {
+    const headers = new Headers()
+    if (version !== null) headers.set('x-api-version', version)
+    return headers
+  }
+
+  it('reloads when effective API version is greater than expected', () => {
+    const location = makeLocation('')
+    maybeReloadForApiVersion(location, makeHeaders('1'))
+    expect(location.replace).toHaveBeenCalledWith(
+      `${dashboardPathname}?api_version_reloaded=1`
+    )
+  })
+
+  it('does not reload when effective API version equals expected', () => {
+    const location = makeLocation('')
+    maybeReloadForApiVersion(location, makeHeaders('0'))
+    expect(location.replace).not.toHaveBeenCalled()
+  })
+
+  it('does not reload when effective API version is less than expected (FE loaded from newer node, cluster not fully updated)', () => {
+    const location = makeLocation('')
+    maybeReloadForApiVersion(location, makeHeaders('-1'))
+    expect(location.replace).not.toHaveBeenCalled()
+  })
+
+  it('does not reload when x-api-version header is absent', () => {
+    const location = makeLocation('')
+    maybeReloadForApiVersion(location, makeHeaders(null))
+    expect(location.replace).not.toHaveBeenCalled()
+  })
+
+  it('does not reload when already reloaded for this version', () => {
+    const location = makeLocation('?api_version_reloaded=1')
+    maybeReloadForApiVersion(location, makeHeaders('1'))
+    expect(location.replace).not.toHaveBeenCalled()
+  })
+
+  it('reloads again if a newer version is detected after a previous reload', () => {
+    const location = makeLocation('?api_version_reloaded=1')
+    maybeReloadForApiVersion(location, makeHeaders('2'))
+    expect(location.replace).toHaveBeenCalledWith(
+      `${dashboardPathname}?api_version_reloaded=2`
+    )
+  })
+})

assets/js/dashboard/util/url-search-params.ts

@@ -19,6 +19,69 @@ const LABEL_URL_PARAM_NAME = 'l'
 
 const REDIRECTED_SEARCH_PARAM_NAME = 'r'
 
+const API_VERSION_RELOAD_PARAM_NAME = 'api_version_reloaded'
+
+const EXPECTED_API_VERSION = parseInt(
+  document
+    .querySelector('meta[name="x-api-version"]')
+    ?.getAttribute('content') ?? '0',
+  10
+)
+
+/**
+ * Navigates to the current URL with `api_version_reloaded=<currentApiVersion>`
+ * appended, using `location.replace` so the pre-reload entry is not kept in
+ * browser history.
+ *
+ * Returns early without navigating if:
+ *
+ * - the x-plausible-version response header is not present
+ * - the expected version matches the actual version
+ * - the version is already present in search params
+ *
+ * The latter prevents an infinite reload loop when the versions are
+ * permanently out of sync.
+ *
+ * BE: lib/plausible_web/plugs/internal_stats_api_version.ex
+ */
+export function maybeReloadForApiVersion(
+  windowLocation: Location,
+  responseHeaders: Headers
+) {
+  const currentApiVersion = getCurrentApiVersion(responseHeaders)
+  const params = new URLSearchParams(windowLocation.search)
+
+  if (
+    currentApiVersion === null ||
+    currentApiVersion <= EXPECTED_API_VERSION ||
+    params.get(API_VERSION_RELOAD_PARAM_NAME) === currentApiVersion.toString()
+  ) {
+    return
+  }
+
+  console.warn('API version mismatch detected, reloading...')
+
+  const newSearch = searchWithApiVersionReload(
+    windowLocation.search,
+    currentApiVersion.toString()
+  )
+  windowLocation.replace(
+    `${windowLocation.pathname}${newSearch}${windowLocation.hash}`
+  )
+}
+
+function getCurrentApiVersion(responseHeaders: Headers): number | null {
+  const versionString = responseHeaders?.get('x-api-version')
+  return versionString ? parseInt(versionString, 10) : null
+}
+
+function searchWithApiVersionReload(search: string, value: string): string {
+  return stringifySearch({
+    ...parseSearch(search),
+    [API_VERSION_RELOAD_PARAM_NAME]: value
+  })
+}
+
 /**
  * This function is able to serialize for URL simple params @see serializeSimpleSearchEntry as well
  * two complex params, labels and filters.

lib/plausible/application.ex

@@ -31,6 +31,7 @@ defmodule Plausible.Application do
     children =
       [
         cluster,
+        Plausible.InternalStatsApiVersion,
         {PartitionSupervisor,
          child_spec: Task.Supervisor, name: Plausible.UserAgentParseTaskSupervisor},
         Plausible.Session.BalancerSupervisor,

lib/plausible/internal_stats_api_version.ex

@@ -0,0 +1,77 @@
+defmodule Plausible.InternalStatsApiVersion do
+  @moduledoc """
+  Tracks the effective internal stats API version across the cluster.
+
+  Increment `@api_version` when deploying a change that breaks dashboards
+  already loaded from a previous deployment. The FE, upon detecting a
+  version mismatch, reloads the page to fetch the new dashboard code.
+
+  Each app node has `@api_version` compiled in. The effective version served
+  to clients is the minimum across all connected nodes, fetched via
+  `:rpc.multicall` and refreshed every 30 seconds. This means the version
+  only advances once every node in a rolling deploy is running the new code,
+  avoiding repeated dashboard reloads during the deployment window.
+  """
+  use GenServer
+
+  @api_version 0
+
+  @refresh_interval :timer.seconds(30)
+
+  @spec api_version() :: non_neg_integer()
+  def api_version, do: @api_version
+
+  @spec effective_version() :: non_neg_integer()
+  def effective_version() do
+    # Use 0 as a placeholder version until the first multicall completes.
+    # The FE only reloads when the received version exceeds its compiled-in
+    # expectation, so 0 is always safe regardless of current @api_version.
+    case :ets.lookup(__MODULE__, :version) do
+      [{:version, v}] -> v
+      _ -> 0
+    end
+  end
+
+  def start_link(opts \\ []) do
+    GenServer.start_link(__MODULE__, opts, name: __MODULE__)
+  end
+
+  @impl GenServer
+  def init(_opts) do
+    __MODULE__ =
+      :ets.new(__MODULE__, [
+        :named_table,
+        :set,
+        :protected,
+        {:read_concurrency, true}
+      ])
+
+    {:ok, nil, {:continue, :fetch}}
+  end
+
+  @impl GenServer
+  def handle_continue(:fetch, state) do
+    Process.send_after(self(), :refresh, @refresh_interval)
+    :ets.insert(__MODULE__, {:version, fetch_cluster_min()})
+    {:noreply, state}
+  end
+
+  @impl GenServer
+  def handle_info(:refresh, state) do
+    Process.send_after(self(), :refresh, @refresh_interval)
+    :ets.insert(__MODULE__, {:version, fetch_cluster_min()})
+    {:noreply, state}
+  end
+
+  defp fetch_cluster_min() do
+    {results, _bad_nodes} = :rpc.multicall(__MODULE__, :api_version, [], :timer.seconds(5))
+    cluster_min(results)
+  end
+
+  def cluster_min(results) do
+    case Enum.filter(results, &is_integer/1) do
+      [] -> @api_version
+      versions -> Enum.min(versions)
+    end
+  end
+end

lib/plausible_web/plugs/internal_stats_api_version.ex

@@ -0,0 +1,18 @@
+defmodule PlausibleWeb.Plugs.InternalStatsApiVersion do
+  @moduledoc """
+  Adds the `x-api-version` response header to all internal
+  stats API responses. See `Plausible.InternalStatsApiVersion`
+  for version tracking and rollout logic.
+  """
+  @behaviour Plug
+  import Plug.Conn
+
+  @impl true
+  def init(opts), do: opts
+
+  @impl true
+  def call(conn, _opts) do
+    version = Plausible.InternalStatsApiVersion.effective_version()
+    put_resp_header(conn, "x-api-version", to_string(version))
+  end
+end