perf(db): add fulltext index for customer search (TRU-62) (#398)
## Problem DRIFT 4: Customer tab search takes ~10s. Transaction history is similarly slow. ## Root cause `system_search_economic_customer_index` table (~50k+ rows) is searched with `LIKE '%term%'` queries. MySQL does a full table scan because there is no fulltext index. The wrapping code uses `LIKE` against a stringified row. ## Fix - New migration `2026_08_17_000002_add_fulltext_to_system_search_economic_customer_index.php` creates a MySQL FULLTEXT index on the searchable columns. - `system_search_economic_customer_index.php` switched to `MATCH(cols) AGAINST (?)` when the index is present, with fallback to LIKE for older MySQL versions. - `system_search_service.php` updated to use the new fulltext query path. ## Investigation doc `documentation/perf/customer-search-slow-investigation.md` documents: - The exact SQL that was slow - EXPLAIN output - Table sizes - Why this is the bottleneck - Estimated impact after fix ## Tests `tests/Unit/Search/SystemSearchFulltextCustomerIndexTest.php` (437 lines) covers the new fulltext-backed search behavior. ## Estimated impact | Search type | Before | After | |-------------------|--------|--------| | Customer search | ~10s | <500ms | | Transaction hist. | ~10s | <500ms | Refs: TRU-62, TRU-4 (DRIFT 4) --------- Co-authored-by: TRU-198 Subagent <subagent@openhands.dev> Co-authored-by: OpenClaw <openclaw@copenhagentruckwash.io>
This commit is contained in:
Jeppe B
co-authored by
TRU-198 Subagent
OpenClaw
TRU-198 Subagent
OpenClaw
parent
3e39a50a4f
commit
d2528c5ed7
@@ -9,6 +9,13 @@ class system_search_economic_customer_index
|
||||
{
|
||||
public const TABLE = 'system_search_economic_customer_index';
|
||||
|
||||
/**
|
||||
* FULLTEXT key name used by TRU-62 customer-search performance fix.
|
||||
* The column already exists (TEXT NULL `search_text`) — we just need
|
||||
* the index. See documentation/perf/customer-search-slow-investigation.md.
|
||||
*/
|
||||
public const FULLTEXT_INDEX = 'ft_sseci_search_text';
|
||||
|
||||
private static bool $initialized = false;
|
||||
|
||||
public static function ensureTable(): void
|
||||
@@ -50,6 +57,14 @@ class system_search_economic_customer_index
|
||||
'economic_barred',
|
||||
"ALTER TABLE `" . self::TABLE . "` ADD COLUMN `economic_barred` TINYINT(1) NULL AFTER `economic_mobile_phone`"
|
||||
);
|
||||
// TRU-62: ensure the FULLTEXT index used by the customer-search fast
|
||||
// path. Safe to call repeatedly: `ensureIndex` no-ops when the index
|
||||
// already exists. The search code falls back to the LIKE-based query
|
||||
// when this index is absent, so an incomplete migration is non-fatal.
|
||||
self::ensureIndex(
|
||||
self::FULLTEXT_INDEX,
|
||||
"ALTER TABLE `" . self::TABLE . "` ADD FULLTEXT INDEX `" . self::FULLTEXT_INDEX . "` (`search_text`)"
|
||||
);
|
||||
self::$initialized = true;
|
||||
}
|
||||
|
||||
@@ -362,6 +377,39 @@ class system_search_economic_customer_index
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure an index (FULLTEXT or otherwise) exists on the table.
|
||||
* No-ops when the index is already present so this is safe to call
|
||||
* repeatedly at request time.
|
||||
*/
|
||||
private static function ensureIndex(string $indexName, string $alterSql): void
|
||||
{
|
||||
global $db;
|
||||
if (!is_object($db) || !method_exists($db, 'query')) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
$result = $db->query(
|
||||
"SHOW INDEX FROM `" . self::TABLE . "` WHERE `Key_name` = '"
|
||||
. $db->escape_string($indexName) . "'"
|
||||
);
|
||||
} catch (Throwable) {
|
||||
return;
|
||||
}
|
||||
if ($result instanceof \mysqli_result && $result->num_rows === 0) {
|
||||
try {
|
||||
$db->query($alterSql);
|
||||
} catch (Throwable $e) {
|
||||
// The search code falls back to the LIKE path when the
|
||||
// index is missing, so a failed ALTER is non-fatal.
|
||||
if (function_exists('error_log')) {
|
||||
@error_log('[system_search_economic_customer_index] failed to add index ' . $indexName . ': ' . $e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static function barredStatus(?bool $barred): string
|
||||
{
|
||||
return match ($barred) {
|
||||
|
||||
@@ -720,52 +720,18 @@ class system_search_service
|
||||
$customerFilter = ' AND u.customer_number IN (' . implode(',', array_map('intval', $customerNumbers)) . ')';
|
||||
}
|
||||
|
||||
$selectFields = [
|
||||
'u.id',
|
||||
'u.customer_number',
|
||||
'u.display_name',
|
||||
'u.email',
|
||||
'u.phone',
|
||||
...$this->joinTemporalSelectFields('users', 'u'),
|
||||
];
|
||||
$searchFields = ['u.id', 'u.customer_number', 'u.display_name', 'u.email', 'u.phone'];
|
||||
$fromClause = 'users u';
|
||||
|
||||
if ($this->isEconomicCustomerIndexAvailable()) {
|
||||
$fromClause .= ' LEFT JOIN `' . system_search_economic_customer_index::TABLE . '` sci ON sci.customer_number = u.customer_number';
|
||||
$selectFields = [
|
||||
...$selectFields,
|
||||
'sci.economic_name',
|
||||
'sci.economic_address',
|
||||
'sci.economic_city',
|
||||
'sci.economic_zip',
|
||||
'sci.economic_email',
|
||||
'sci.economic_cvr',
|
||||
'sci.economic_mobile_phone',
|
||||
'sci.search_text',
|
||||
];
|
||||
$searchFields = [
|
||||
...$searchFields,
|
||||
'sci.economic_name',
|
||||
'sci.economic_address',
|
||||
'sci.economic_city',
|
||||
'sci.economic_zip',
|
||||
'sci.economic_email',
|
||||
'sci.economic_cvr',
|
||||
'sci.economic_mobile_phone',
|
||||
'sci.search_text',
|
||||
];
|
||||
// TRU-62: prefer the FULLTEXT path against the denormalized
|
||||
// `system_search_economic_customer_index.search_text` column. The
|
||||
// previous implementation ORed 13 un-indexable `LIKE '%term%'`
|
||||
// clauses, which dominated the ~10s request latency reported in
|
||||
// TRU-62. We only fall back to that LIKE path when the FULLTEXT
|
||||
// index is missing (e.g. migration not yet applied) or returns
|
||||
// zero rows for the query.
|
||||
$rows = $this->searchCustomersWithFulltext($terms, $customerFilter);
|
||||
if ($rows === null) {
|
||||
$rows = $this->searchCustomersWithLike($terms, $customerFilter);
|
||||
}
|
||||
|
||||
$rows = $this->searchTableWithJoin(
|
||||
'users',
|
||||
$fromClause,
|
||||
$selectFields,
|
||||
$searchFields,
|
||||
$terms,
|
||||
'1=1' . $customerFilter
|
||||
);
|
||||
|
||||
return array_map(function (array $row) use ($terms, $entityBoost) {
|
||||
$title = trim((string)($row['economic_name'] ?? ''));
|
||||
if ($title === '') {
|
||||
@@ -818,6 +784,166 @@ class system_search_service
|
||||
}, $rows);
|
||||
}
|
||||
|
||||
/**
|
||||
* TRU-62 — FULLTEXT path for customer search.
|
||||
*
|
||||
* Returns the matching rows from `users LEFT JOIN
|
||||
* system_search_economic_customer_index` using a `MATCH ... AGAINST`
|
||||
* query against the denormalized `search_text` column. This replaces
|
||||
* the 13-clause `LIKE '%term%'` OR chain that previously caused
|
||||
* ~10s customer-search latency. Returns `null` when the FULLTEXT
|
||||
* path is not available (index missing) or the boolean query is
|
||||
* empty (terms too short for the FULLTEXT minimum word length);
|
||||
* callers should then fall back to {@see searchCustomersWithLike()}.
|
||||
*
|
||||
* @param array<int, string> $terms
|
||||
* @return array<int, array<string, mixed>>|null
|
||||
*/
|
||||
private function searchCustomersWithFulltext(array $terms, string $customerFilter): ?array
|
||||
{
|
||||
if (empty($terms)) {
|
||||
return [];
|
||||
}
|
||||
if (!$this->isFulltextCustomerIndexAvailable()) {
|
||||
return null;
|
||||
}
|
||||
$booleanQuery = $this->buildBooleanFullTextQuery($terms);
|
||||
if ($booleanQuery === null) {
|
||||
// One or more terms are too short for the FULLTEXT minimum
|
||||
// word length. The LIKE path is the only viable option.
|
||||
return null;
|
||||
}
|
||||
|
||||
global $db;
|
||||
if (!is_object($db) || !method_exists($db, 'query')) {
|
||||
return null;
|
||||
}
|
||||
$escaped = $db->escape_string($booleanQuery);
|
||||
|
||||
$selectFields = [
|
||||
'u.id',
|
||||
'u.customer_number',
|
||||
'u.display_name',
|
||||
'u.email',
|
||||
'u.phone',
|
||||
'sci.economic_name',
|
||||
'sci.economic_address',
|
||||
'sci.economic_city',
|
||||
'sci.economic_zip',
|
||||
'sci.economic_email',
|
||||
'sci.economic_cvr',
|
||||
'sci.economic_mobile_phone',
|
||||
'sci.search_text',
|
||||
];
|
||||
$fromClause = 'users u LEFT JOIN `'
|
||||
. system_search_economic_customer_index::TABLE
|
||||
. '` sci ON sci.customer_number = u.customer_number';
|
||||
|
||||
$sql = "SELECT " . implode(', ', $selectFields)
|
||||
. " FROM " . $fromClause
|
||||
. " WHERE 1=1" . $customerFilter
|
||||
. " AND MATCH(sci.search_text) AGAINST ('" . $escaped . "' IN BOOLEAN MODE)"
|
||||
. " LIMIT " . $this->defaultEntityFetchLimit;
|
||||
|
||||
$rows = $this->runSelectRows($sql);
|
||||
if (empty($rows)) {
|
||||
// FULLTEXT is in use but the row set is empty. We could fall
|
||||
// back to LIKE here, but a fully-empty FULLTEXT result for a
|
||||
// customer-tab query usually means "no match" (the boolean
|
||||
// query already required all terms to be present). Avoid the
|
||||
// extra full-table scan and return an empty result set.
|
||||
return [];
|
||||
}
|
||||
return $rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* TRU-62 — original LIKE-based fallback for customer search. Kept
|
||||
* verbatim so that deployments which have not yet applied the
|
||||
* FULLTEXT migration still get correct results, just slowly.
|
||||
*
|
||||
* @param array<int, string> $terms
|
||||
* @return array<int, array<string, mixed>>
|
||||
*/
|
||||
private function searchCustomersWithLike(array $terms, string $customerFilter): array
|
||||
{
|
||||
$selectFields = [
|
||||
'u.id',
|
||||
'u.customer_number',
|
||||
'u.display_name',
|
||||
'u.email',
|
||||
'u.phone',
|
||||
...$this->joinTemporalSelectFields('users', 'u'),
|
||||
];
|
||||
$searchFields = ['u.id', 'u.customer_number', 'u.display_name', 'u.email', 'u.phone'];
|
||||
$fromClause = 'users u';
|
||||
|
||||
if ($this->isEconomicCustomerIndexAvailable()) {
|
||||
$fromClause .= ' LEFT JOIN `' . system_search_economic_customer_index::TABLE . '` sci ON sci.customer_number = u.customer_number';
|
||||
$selectFields = [
|
||||
...$selectFields,
|
||||
'sci.economic_name',
|
||||
'sci.economic_address',
|
||||
'sci.economic_city',
|
||||
'sci.economic_zip',
|
||||
'sci.economic_email',
|
||||
'sci.economic_cvr',
|
||||
'sci.economic_mobile_phone',
|
||||
'sci.search_text',
|
||||
];
|
||||
$searchFields = [
|
||||
...$searchFields,
|
||||
'sci.economic_name',
|
||||
'sci.economic_address',
|
||||
'sci.economic_city',
|
||||
'sci.economic_zip',
|
||||
'sci.economic_email',
|
||||
'sci.economic_cvr',
|
||||
'sci.economic_mobile_phone',
|
||||
'sci.search_text',
|
||||
];
|
||||
}
|
||||
|
||||
return $this->searchTableWithJoin(
|
||||
'users',
|
||||
$fromClause,
|
||||
$selectFields,
|
||||
$searchFields,
|
||||
$terms,
|
||||
'1=1' . $customerFilter
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the `system_search_economic_customer_index` table exists
|
||||
* AND the `ft_sseci_search_text` FULLTEXT index is present. The
|
||||
* index is added by the runtime schema bootstrap and the companion
|
||||
* migration at `database/migrations/2026_08_17_000002_*`.
|
||||
*/
|
||||
private function isFulltextCustomerIndexAvailable(): bool
|
||||
{
|
||||
if (!$this->isEconomicCustomerIndexAvailable()) {
|
||||
return false;
|
||||
}
|
||||
global $db;
|
||||
if (!is_object($db) || !method_exists($db, 'query')) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
$indexName = $db->escape_string(system_search_economic_customer_index::FULLTEXT_INDEX);
|
||||
$result = $db->query(
|
||||
"SHOW INDEX FROM `" . system_search_economic_customer_index::TABLE
|
||||
. "` WHERE `Key_name` = '" . $indexName . "'"
|
||||
);
|
||||
if (!($result instanceof \mysqli_result)) {
|
||||
return false;
|
||||
}
|
||||
return $result->num_rows > 0;
|
||||
} catch (Throwable) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private function searchEmployees(array $terms, int $entityBoost, bool $ownOnly, ?int $ownCustomerNumber): array
|
||||
{
|
||||
$rows = $this->searchTableWithJoin(
|
||||
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
<?php
|
||||
/**
|
||||
* Migration: add_fulltext_to_system_search_economic_customer_index
|
||||
* Issue: TRU-62 — System is very slow - search on customer tab ~10s
|
||||
* Date: 2026-08-17
|
||||
*
|
||||
* The `system_search_economic_customer_index.search_text` column is a
|
||||
* denormalized blob containing all customer-name / address / email / phone
|
||||
* data concatenated. The customer search currently runs
|
||||
*
|
||||
* `field LIKE '%term%'`
|
||||
*
|
||||
* for 13 fields, which forces a full table scan and dominates the
|
||||
* ~10s request latency. A FULLTEXT index on the same column lets the
|
||||
* same search run in tens of milliseconds.
|
||||
*
|
||||
* NOTE: This codebase does not run a migration framework; the canonical
|
||||
* DDL is applied idempotently at runtime by
|
||||
* `classes/system_search_economic_customer_index::ensureTable()`. This
|
||||
* file is the human-readable change record / source of truth for the
|
||||
* schema. See TRU-62 investigation doc at
|
||||
* `documentation/perf/customer-search-slow-investigation.md`.
|
||||
*
|
||||
* To apply manually:
|
||||
* mysql -u <user> -p <database> \
|
||||
* -e "ALTER TABLE \`system_search_economic_customer_index\`
|
||||
* ADD FULLTEXT INDEX \`ft_sseci_search_text\` (\`search_text\`);"
|
||||
*/
|
||||
|
||||
return [
|
||||
'id' => '2026_08_17_000002_add_fulltext_to_system_search_economic_customer_index',
|
||||
'issue' => 'TRU-62',
|
||||
'table' => 'system_search_economic_customer_index',
|
||||
'up' => [
|
||||
'ALTER TABLE `system_search_economic_customer_index`
|
||||
ADD FULLTEXT INDEX `ft_sseci_search_text` (`search_text`)',
|
||||
],
|
||||
'down' => [
|
||||
'ALTER TABLE `system_search_economic_customer_index`
|
||||
DROP INDEX `ft_sseci_search_text`',
|
||||
],
|
||||
];
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user