From 5be31bee1c7a81cbfe3e70a872818a1e9fef7b5b Mon Sep 17 00:00:00 2001 From: Jeppe Bundgaard Date: Fri, 13 Mar 2026 00:59:40 +0100 Subject: [PATCH] Extend system search to include local e-conomic customer index fields, synonym expansion (e.g., `rabat` -> `discount`), and enhanced entity matching. Update OpenAPI spec, unit tests, and cron sync tasks accordingly. --- openapi.yaml | 8 +- .../app/classes/system_search_service.php | 286 ++++++++++++++++-- services/nginx/app/cron/Cron.php | 36 ++- .../Search/SystemSearchOpenApiSpecTest.php | 7 + .../SystemSearchServiceIntentFlowTest.php | 22 ++ 5 files changed, 326 insertions(+), 33 deletions(-) diff --git a/openapi.yaml b/openapi.yaml index 66b16985..0c354c13 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -7751,7 +7751,7 @@ paths: tags: - Search summary: System-wide search - description: Search across all supported entities with permission-aware filtering and optional intent parsing debug metadata. + description: Search across all supported entities with permission-aware filtering and optional intent parsing debug metadata. Customer records and customer-related entities are matched against a local e-conomic customer index (name/address/email/CVR) that is refreshed by cron. operationId: systemWideSearchGet parameters: - in: query @@ -7759,7 +7759,7 @@ paths: required: true schema: type: string - description: Free-text query to search for. + description: Free-text query to search for. Supports natural-language intent fallback and domain synonyms such as `rabat` -> `discount`. - in: query name: include_types required: false @@ -7826,7 +7826,7 @@ paths: tags: - Search summary: System-wide search - description: Search across all supported entities using JSON request payload. + description: Search across all supported entities using JSON request payload. Customer-related matches include local e-conomic customer index fields. operationId: systemWideSearchPost requestBody: required: true @@ -9224,7 +9224,7 @@ components: properties: query: type: string - description: Free-text query to search for. + description: Free-text query to search for. Customer lookups include local e-conomic index fields and lexical synonym expansion (for example `rabat` -> `discount`). include_types: type: array items: diff --git a/services/nginx/app/classes/system_search_service.php b/services/nginx/app/classes/system_search_service.php index c26863d1..bda135df 100644 --- a/services/nginx/app/classes/system_search_service.php +++ b/services/nginx/app/classes/system_search_service.php @@ -18,6 +18,11 @@ class system_search_service public function __construct(?system_search_intent_parser_i $intentParser = null) { $this->intentParser = $intentParser ?? new system_search_openai_intent_parser(); + try { + system_search_economic_customer_index::ensureTable(); + } catch (Throwable) { + // Search should work even if index bootstrap is temporarily unavailable. + } } public function search(array $options): array @@ -93,7 +98,7 @@ class system_search_service return $cached; } - $terms = $this->limitTerms($this->tokenize($query)); + $terms = $this->buildExpandedTerms($this->tokenize($query)); $entityBoost = []; $initialResults = $this->executeLexicalSearch( $activeTypes, @@ -127,7 +132,7 @@ class system_search_service if (!empty($intent['success'])) { $intentMeta['status'] = 'ok'; - $expandedTerms = $this->limitTerms([ + $expandedTerms = $this->buildExpandedTerms([ ...$terms, ...$this->tokenize((string)($intent['normalized_query'] ?? '')), ...$this->tokenize(implode(' ', (array)($intent['aliases'] ?? []))), @@ -322,30 +327,100 @@ class system_search_service private function searchCustomers(array $terms, int $entityBoost, bool $ownOnly, ?int $ownCustomerNumber, array $forcedCustomerNumbers): array { - $customerNumbers = !empty($forcedCustomerNumbers) ? $forcedCustomerNumbers : ($ownOnly && $ownCustomerNumber ? [$ownCustomerNumber] : []); - $rows = $this->searchTable( + $customerNumbers = !empty($forcedCustomerNumbers) + ? $forcedCustomerNumbers + : (($ownOnly && $ownCustomerNumber) ? [$ownCustomerNumber] : []); + $customerFilter = ''; + if (!empty($customerNumbers)) { + $customerFilter = ' AND u.customer_number IN (' . implode(',', array_map('intval', $customerNumbers)) . ')'; + } + + $selectFields = ['u.id', 'u.customer_number', 'u.display_name', 'u.email', 'u.phone']; + $searchFields = ['u.id', 'u.customer_number', 'u.display_name', 'u.email', 'u.phone']; + $fromClause = 'users u'; + + if ($this->isEconomicCustomerIndexAvailable()) { + $fromClause .= ' LEFT JOIN `' . system_search_economic_customer_index::TABLE . '` sci ON sci.customer_number = u.customer_number'; + $selectFields = [ + ...$selectFields, + 'sci.economic_name', + 'sci.economic_address', + 'sci.economic_city', + 'sci.economic_zip', + 'sci.economic_email', + 'sci.economic_cvr', + 'sci.economic_mobile_phone', + 'sci.search_text', + ]; + $searchFields = [ + ...$searchFields, + 'sci.economic_name', + 'sci.economic_address', + 'sci.economic_city', + 'sci.economic_zip', + 'sci.economic_email', + 'sci.economic_cvr', + 'sci.economic_mobile_phone', + 'sci.search_text', + ]; + } + + $rows = $this->searchTableWithJoin( 'users', - ['id', 'customer_number', 'display_name', 'email', 'phone'], - ['id', 'customer_number', 'display_name', 'email', 'phone'], + $fromClause, + $selectFields, + $searchFields, $terms, - $customerNumbers, - 'customer_number' + '1=1' . $customerFilter ); return array_map(function (array $row) use ($terms, $entityBoost) { + $title = trim((string)($row['economic_name'] ?? '')); + if ($title === '') { + $title = trim((string)($row['display_name'] ?? '')); + } + if ($title === '') { + $title = 'Customer #' . (string)($row['customer_number'] ?? ''); + } + + $description = trim((string)($row['email'] ?? '')); + if ($description === '') { + $description = trim((string)($row['economic_email'] ?? '')); + } + return [ 'entity_type' => 'customers', 'entity_id' => (string)$row['id'], - 'title' => (string)($row['display_name'] ?: ('Customer #' . $row['customer_number'])), - 'description' => (string)($row['email'] ?? ''), + 'title' => $title, + 'description' => $description, 'customer_number' => isset($row['customer_number']) ? (int)$row['customer_number'] : null, - 'score' => $this->scoreRow($row, ['customer_number', 'display_name', 'email', 'phone'], $terms) + $entityBoost, + 'score' => $this->scoreRow($row, [ + 'customer_number', + 'display_name', + 'email', + 'phone', + 'economic_name', + 'economic_address', + 'economic_city', + 'economic_zip', + 'economic_email', + 'economic_cvr', + 'economic_mobile_phone', + 'search_text', + ], $terms) + $entityBoost, 'payload' => [ 'id' => (int)$row['id'], 'customer_number' => isset($row['customer_number']) ? (int)$row['customer_number'] : null, 'display_name' => $row['display_name'] ?? null, 'email' => $row['email'] ?? null, 'phone' => $row['phone'] ?? null, + 'economic_name' => $row['economic_name'] ?? null, + 'economic_address' => $row['economic_address'] ?? null, + 'economic_city' => $row['economic_city'] ?? null, + 'economic_zip' => $row['economic_zip'] ?? null, + 'economic_email' => $row['economic_email'] ?? null, + 'economic_cvr' => $row['economic_cvr'] ?? null, + 'economic_mobile_phone' => $row['economic_mobile_phone'] ?? null, ], ]; }, $rows); @@ -552,28 +627,79 @@ class system_search_service $customerFilter = ' AND u.customer_number = ' . (int)$ownCustomerNumber; } + $fromClause = 'price_overrides po INNER JOIN users u ON u.id = po.user_id'; + $selectFields = ['po.id', 'po.user_id', 'po.is_category', 'po.product_or_category_id', 'po.percentage', 'u.customer_number', 'u.display_name']; + $searchFields = ['po.id', 'po.user_id', 'po.product_or_category_id', 'po.percentage', 'u.customer_number', 'u.display_name']; + + if ($this->isEconomicCustomerIndexAvailable()) { + $fromClause .= ' LEFT JOIN `' . system_search_economic_customer_index::TABLE . '` sci ON sci.customer_number = u.customer_number'; + $selectFields = [ + ...$selectFields, + 'sci.economic_name', + 'sci.economic_address', + 'sci.economic_city', + 'sci.economic_zip', + 'sci.economic_email', + 'sci.economic_cvr', + 'sci.economic_mobile_phone', + 'sci.search_text', + ]; + $searchFields = [ + ...$searchFields, + 'sci.economic_name', + 'sci.economic_address', + 'sci.economic_city', + 'sci.economic_zip', + 'sci.economic_email', + 'sci.economic_cvr', + 'sci.economic_mobile_phone', + 'sci.search_text', + ]; + } + $rows = $this->searchTableWithJoin( 'price_overrides', - 'price_overrides po INNER JOIN users u ON u.id = po.user_id', - ['po.id', 'po.user_id', 'po.is_category', 'po.product_or_category_id', 'po.percentage', 'u.customer_number', 'u.display_name'], - ['po.id', 'po.user_id', 'po.product_or_category_id', 'po.percentage', 'u.customer_number', 'u.display_name'], + $fromClause, + $selectFields, + $searchFields, $terms, '1=1' . $customerFilter ); return array_map(function (array $row) use ($terms, $entityBoost) { + $customerDisplay = trim((string)($row['economic_name'] ?? '')); + if ($customerDisplay === '') { + $customerDisplay = trim((string)($row['display_name'] ?? '')); + } return [ 'entity_type' => 'customer_discounts', 'entity_id' => (string)$row['id'], 'title' => 'Discount #' . (string)$row['id'], - 'description' => (string)('Customer ' . ($row['customer_number'] ?? '') . ' / ' . ($row['display_name'] ?? '')), + 'description' => (string)('Customer ' . ($row['customer_number'] ?? '') . ' / ' . $customerDisplay), 'customer_number' => isset($row['customer_number']) ? (int)$row['customer_number'] : null, - 'score' => $this->scoreRow($row, ['id', 'customer_number', 'display_name', 'product_or_category_id', 'percentage', 'user_id'], $terms) + $entityBoost, + 'score' => $this->scoreRow($row, [ + 'id', + 'customer_number', + 'display_name', + 'economic_name', + 'economic_address', + 'economic_city', + 'economic_zip', + 'economic_email', + 'economic_cvr', + 'economic_mobile_phone', + 'search_text', + 'product_or_category_id', + 'percentage', + 'user_id', + ], $terms) + $entityBoost, 'payload' => [ 'id' => (int)$row['id'], 'customer_number' => isset($row['customer_number']) ? (int)$row['customer_number'] : null, 'product_or_category_id' => $row['product_or_category_id'] ?? null, 'percentage' => isset($row['percentage']) ? (int)$row['percentage'] : null, + 'economic_name' => $row['economic_name'] ?? null, + 'economic_cvr' => $row['economic_cvr'] ?? null, ], ]; }, $rows); @@ -581,29 +707,85 @@ class system_search_service private function searchCustomerFixedPrices(array $terms, int $entityBoost, bool $ownOnly, ?int $ownCustomerNumber, array $forcedCustomerNumbers): array { - $customerNumbers = !empty($forcedCustomerNumbers) ? $forcedCustomerNumbers : ($ownOnly && $ownCustomerNumber ? [$ownCustomerNumber] : []); - $rows = $this->searchTable( + $customerNumbers = !empty($forcedCustomerNumbers) + ? $forcedCustomerNumbers + : (($ownOnly && $ownCustomerNumber) ? [$ownCustomerNumber] : []); + $customerFilter = ''; + if (!empty($customerNumbers)) { + $customerFilter = ' AND cfp.customer_number IN (' . implode(',', array_map('intval', $customerNumbers)) . ')'; + } + + $fromClause = 'customer_fixed_pricing cfp'; + $selectFields = ['cfp.id', 'cfp.customer_number', 'cfp.price', 'cfp.description']; + $searchFields = ['cfp.id', 'cfp.customer_number', 'cfp.price', 'cfp.description']; + + if ($this->isEconomicCustomerIndexAvailable()) { + $fromClause .= ' LEFT JOIN `' . system_search_economic_customer_index::TABLE . '` sci ON sci.customer_number = cfp.customer_number'; + $selectFields = [ + ...$selectFields, + 'sci.economic_name', + 'sci.economic_address', + 'sci.economic_city', + 'sci.economic_zip', + 'sci.economic_email', + 'sci.economic_cvr', + 'sci.economic_mobile_phone', + 'sci.search_text', + ]; + $searchFields = [ + ...$searchFields, + 'sci.economic_name', + 'sci.economic_address', + 'sci.economic_city', + 'sci.economic_zip', + 'sci.economic_email', + 'sci.economic_cvr', + 'sci.economic_mobile_phone', + 'sci.search_text', + ]; + } + + $rows = $this->searchTableWithJoin( 'customer_fixed_pricing', - ['id', 'customer_number', 'price', 'description'], - ['id', 'customer_number', 'price', 'description'], + $fromClause, + $selectFields, + $searchFields, $terms, - $customerNumbers, - 'customer_number' + '1=1' . $customerFilter ); return array_map(function (array $row) use ($terms, $entityBoost) { + $description = trim((string)($row['description'] ?? '')); + if ($description === '') { + $description = trim((string)($row['economic_name'] ?? '')); + } return [ 'entity_type' => 'customer_fixed_prices', 'entity_id' => (string)$row['id'], 'title' => 'Fixed pricing #' . (string)$row['id'], - 'description' => (string)($row['description'] ?? ''), + 'description' => $description, 'customer_number' => isset($row['customer_number']) ? (int)$row['customer_number'] : null, - 'score' => $this->scoreRow($row, ['id', 'customer_number', 'price', 'description'], $terms) + $entityBoost, + 'score' => $this->scoreRow($row, [ + 'id', + 'customer_number', + 'price', + 'description', + 'economic_name', + 'economic_address', + 'economic_city', + 'economic_zip', + 'economic_email', + 'economic_cvr', + 'economic_mobile_phone', + 'search_text', + ], $terms) + $entityBoost, 'payload' => [ 'id' => (int)$row['id'], 'customer_number' => isset($row['customer_number']) ? (int)$row['customer_number'] : null, 'price' => isset($row['price']) ? (int)$row['price'] : null, 'description' => $row['description'] ?? null, + 'economic_name' => $row['economic_name'] ?? null, + 'economic_cvr' => $row['economic_cvr'] ?? null, ], ]; }, $rows); @@ -1292,6 +1474,58 @@ class system_search_service return array_values($indexed); } + /** + * @param array $terms + * @return array + */ + private function buildExpandedTerms(array $terms): array + { + $base = $this->limitTerms($terms); + if (empty($base)) { + return []; + } + return $this->limitTerms([ + ...$base, + ...$this->expandLexicalSynonyms($base), + ]); + } + + /** + * @param array $terms + * @return array + */ + private function expandLexicalSynonyms(array $terms): array + { + $synonyms = [ + 'rabat' => ['discount', 'discounts'], + 'rabatordning' => ['discount'], + 'rabatter' => ['discounts', 'discount'], + 'discount' => ['rabat'], + 'discounts' => ['rabat'], + 'kunde' => ['customer', 'customers'], + 'kunder' => ['customer', 'customers'], + 'faktura' => ['invoice', 'invoices'], + 'fakturaer' => ['invoice', 'invoices'], + ]; + + $expanded = []; + foreach ($terms as $term) { + $term = trim(mb_strtolower((string)$term)); + if ($term === '' || !isset($synonyms[$term])) { + continue; + } + foreach ($synonyms[$term] as $synonym) { + $expanded[] = $synonym; + } + } + return array_values(array_unique($expanded)); + } + + private function isEconomicCustomerIndexAvailable(): bool + { + return $this->tableExists(system_search_economic_customer_index::TABLE); + } + private function tokenize(string $query): array { $query = trim(mb_strtolower($query)); @@ -1399,14 +1633,14 @@ class system_search_service private function taxonomy(array $activeTypes): array { $aliases = [ - 'customers' => ['customer', 'account', 'company'], + 'customers' => ['customer', 'account', 'company', 'kunde'], 'orders' => ['order', 'work order'], 'order_items' => ['order item', 'line item'], 'invoices' => ['invoice', 'billing'], 'vehicles' => ['vehicle', 'truck', 'plate'], 'employees' => ['employee', 'staff'], 'subusers' => ['subuser', 'driver'], - 'customer_discounts' => ['discount', 'price override'], + 'customer_discounts' => ['discount', 'price override', 'rabat'], 'customer_fixed_prices' => ['fixed price', 'monthly agreement'], 'departments' => ['department', 'location'], 'permissions' => ['permission', 'acl'], diff --git a/services/nginx/app/cron/Cron.php b/services/nginx/app/cron/Cron.php index 867f7fe6..cb1bf401 100644 --- a/services/nginx/app/cron/Cron.php +++ b/services/nginx/app/cron/Cron.php @@ -4,6 +4,7 @@ use classes\backup_store; use classes\economic; use classes\system_search_cache; +use classes\system_search_economic_customer_index; use classes\xlvask; use classes\slack as Slack; use classes\email as Email; @@ -63,6 +64,12 @@ $cron_tasks = [ 'next_run' => 0, 'function' => 'SyncUserEconomicCustomerDetails', ], + 'SyncSystemSearchEconomicCustomerIndex' => [ + 'interval' => 900, // 15 minutes + 'last_run' => 0, + 'next_run' => 0, + 'function' => 'SyncSystemSearchEconomicCustomerIndex', + ], 'backup' => [ 'interval' => 43200, // 12 hours 'last_run' => 0, @@ -134,9 +141,28 @@ function SyncUserEconomicCustomerDiscounts(): void function SyncUserEconomicCustomerDetails(): void { - $users_o = new users_o(); - $users_o->clearAllUsersEconomicCustomerDetailsFromCache(); - //$users_o->syncAllUsersEconomicCustomerDetails(); + try { + $users_o = new users_o(); + $users_o->clearAllUsersEconomicCustomerDetailsFromCache(); + $users_o->syncAllUsersEconomicCustomerDetails(); + $stats = system_search_economic_customer_index::refreshIndex(false); + echo "[" . date('Y-m-d H:i:s') . "][CRON] Refreshed e-conomic customer snapshots and search index. Upserted: " + . (int)($stats['upserted'] ?? 0) . "\n"; + } catch (Throwable $e) { + warn('SyncUserEconomicCustomerDetails failed: ' . $e->getMessage()); + } +} + +function SyncSystemSearchEconomicCustomerIndex(): void +{ + try { + $stats = system_search_economic_customer_index::refreshIndex(false); + echo "[" . date('Y-m-d H:i:s') . "][CRON] Synced system search e-conomic customer index. Processed: " + . (int)($stats['processed'] ?? 0) . ", upserted: " . (int)($stats['upserted'] ?? 0) + . ", deleted: " . (int)($stats['deleted'] ?? 0) . "\n"; + } catch (Throwable $e) { + warn('SyncSystemSearchEconomicCustomerIndex failed: ' . $e->getMessage()); + } } /** @@ -178,12 +204,16 @@ function SystemSearchCacheMaintenanceCron(): void if ($rebuildRequest !== null) { system_search_cache::clearQueryCaches(); system_search_cache::clearIntentCaches(); + system_search_economic_customer_index::refreshIndex(false); echo "[" . date('Y-m-d H:i:s') . "][CRON] System search cache rebuild handled. Scope: " . ($rebuildRequest['scope'] ?? 'all') . "\n"; return; } if (!empty($dirtyTables)) { system_search_cache::clearQueryCaches(); + if (in_array('users', $dirtyTables, true)) { + system_search_economic_customer_index::refreshIndex(false); + } echo "[" . date('Y-m-d H:i:s') . "][CRON] System search query cache invalidated for dirty tables: " . implode(', ', $dirtyTables) . "\n"; } } catch (Throwable $e) { diff --git a/services/nginx/app/tests/Unit/Search/SystemSearchOpenApiSpecTest.php b/services/nginx/app/tests/Unit/Search/SystemSearchOpenApiSpecTest.php index a10fb318..98ca5c60 100644 --- a/services/nginx/app/tests/Unit/Search/SystemSearchOpenApiSpecTest.php +++ b/services/nginx/app/tests/Unit/Search/SystemSearchOpenApiSpecTest.php @@ -41,3 +41,10 @@ it('documents debug_intent and parser metadata schema in openapi', function (): expect($content)->toContain('intent_parser:'); expect($content)->toContain('SystemSearchResponse:'); }); + +it('documents e-conomic indexed customer matching and synonym behavior', function (): void { + $content = system_search_openapi_content_or_skip(); + + expect($content)->toContain('local e-conomic customer index'); + expect($content)->toContain('`rabat` -> `discount`'); +}); diff --git a/services/nginx/app/tests/Unit/Search/SystemSearchServiceIntentFlowTest.php b/services/nginx/app/tests/Unit/Search/SystemSearchServiceIntentFlowTest.php index 7b7b37d0..601a3561 100644 --- a/services/nginx/app/tests/Unit/Search/SystemSearchServiceIntentFlowTest.php +++ b/services/nginx/app/tests/Unit/Search/SystemSearchServiceIntentFlowTest.php @@ -348,3 +348,25 @@ it('caps AI-driven expanded terms to prevent query amplification', function (): } expect($maxLen)->toBeLessThanOrEqual(64); }); + +it('expands danish discount wording into lexical discount synonyms', function (): void { + $parser = new FakeSystemSearchIntentParser(); + $service = new TestableSystemSearchService($parser, [[ + ['entity_type' => 'customer_discounts', 'entity_id' => '1', 'title' => 'd1', 'score' => 95], + ['entity_type' => 'customer_discounts', 'entity_id' => '2', 'title' => 'd2', 'score' => 92], + ['entity_type' => 'customer_discounts', 'entity_id' => '3', 'title' => 'd3', 'score' => 90], + ['entity_type' => 'customer_discounts', 'entity_id' => '4', 'title' => 'd4', 'score' => 88], + ['entity_type' => 'customer_discounts', 'entity_id' => '5', 'title' => 'd5', 'score' => 86], + ]]); + + $service->search([ + 'query' => 'pleno rabat', + 'allowed_types' => ['customer_discounts'], + 'include_associations' => false, + ]); + + $terms = $service->lexicalCalls[0]['terms'] ?? []; + expect($parser->calls)->toBe(0); + expect($terms)->toContain('rabat'); + expect($terms)->toContain('discount'); +});