From 6e887b315fe97af491e0bf5b7731cab7479ae022 Mon Sep 17 00:00:00 2001 From: Jeppe Bundgaard Date: Thu, 12 Mar 2026 20:14:06 +0100 Subject: [PATCH] Introduce a centralized search service with OpenAI-powered intent parsing and caching --- openapi.yaml | 425 +++++++ .../nginx/app/classes/object_property.php | 10 +- .../nginx/app/classes/system_search_cache.php | 234 ++++ .../system_search_openai_intent_parser.php | 247 ++++ .../app/classes/system_search_service.php | 1062 +++++++++++++++++ services/nginx/app/cron/Cron.php | 31 +- .../system_search_intent_parser_i.php | 26 + .../nginx/app/routes/systemSearchRoute.php | 416 +++++++ .../SystemSearchCacheIntegrationTest.php | 157 +++ .../SystemSearchInvalidationHooksTest.php | 32 + .../SystemSearchOpenAiIntentParserTest.php | 192 +++ .../Search/SystemSearchOpenApiSpecTest.php | 38 + .../Search/SystemSearchRouteWiringTest.php | 37 + .../SystemSearchServiceIntentFlowTest.php | 224 ++++ services/nginx/app/traits/db_object_t.php | 33 +- .../app/traits/module_config_variable_t.php | 5 +- 16 files changed, 3164 insertions(+), 5 deletions(-) create mode 100644 services/nginx/app/classes/system_search_cache.php create mode 100644 services/nginx/app/classes/system_search_openai_intent_parser.php create mode 100644 services/nginx/app/classes/system_search_service.php create mode 100644 services/nginx/app/interfaces/system_search_intent_parser_i.php create mode 100644 services/nginx/app/routes/systemSearchRoute.php create mode 100644 services/nginx/app/tests/Integration/Search/SystemSearchCacheIntegrationTest.php create mode 100644 services/nginx/app/tests/Unit/Search/SystemSearchInvalidationHooksTest.php create mode 100644 services/nginx/app/tests/Unit/Search/SystemSearchOpenAiIntentParserTest.php create mode 100644 services/nginx/app/tests/Unit/Search/SystemSearchOpenApiSpecTest.php create mode 100644 services/nginx/app/tests/Unit/Search/SystemSearchRouteWiringTest.php create mode 100644 services/nginx/app/tests/Unit/Search/SystemSearchServiceIntentFlowTest.php diff --git a/openapi.yaml b/openapi.yaml index 258800a5..2c670b1b 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -38,6 +38,8 @@ tags: description: User and employee authentication endpoints - name: Users description: User management and customer operations + - name: Search + description: System-wide search endpoints - name: Orders description: Order creation, management, and retrieval - name: Order Items @@ -7744,6 +7746,152 @@ paths: application/json: schema: {} + /search/system: + get: + tags: + - Search + summary: System-wide search + description: Search across all supported entities with permission-aware filtering and optional intent parsing debug metadata. + operationId: systemWideSearchGet + parameters: + - in: query + name: query + required: true + schema: + type: string + description: Free-text query to search for. + - in: query + name: include_types + required: false + schema: + type: array + items: + $ref: '#/components/schemas/SystemSearchEntityType' + style: form + explode: false + description: Comma-separated list of entity types to include. Defaults to all allowed types. + - in: query + name: exclude_types + required: false + schema: + type: array + items: + $ref: '#/components/schemas/SystemSearchEntityType' + style: form + explode: false + description: Comma-separated list of entity types to exclude. + - in: query + name: include_associations + required: false + schema: + type: boolean + default: true + description: Include associated objects when matching a primary entity such as a customer. + - in: query + name: debug_intent + required: false + schema: + type: boolean + default: false + description: Include intent parser diagnostics in `meta.intent_parser`. + - in: query + name: limit + required: false + schema: + type: integer + minimum: 1 + maximum: 200 + default: 50 + - in: query + name: offset + required: false + schema: + type: integer + minimum: 0 + default: 0 + responses: + '200': + description: Search results returned successfully + content: + application/json: + schema: + $ref: '#/components/schemas/SystemSearchResponse' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + post: + tags: + - Search + summary: System-wide search + description: Search across all supported entities using JSON request payload. + operationId: systemWideSearchPost + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/SystemSearchRequest' + responses: + '200': + description: Search results returned successfully + content: + application/json: + schema: + $ref: '#/components/schemas/SystemSearchResponse' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + + /superuser/search/system/cache: + delete: + tags: + - Search + summary: Clear system search caches + description: Clears both query-result cache and intent-parser cache namespaces for system-wide search. + operationId: clearSystemSearchCache + responses: + '200': + description: Cache cleared successfully + content: + application/json: + schema: + $ref: '#/components/schemas/SystemSearchCacheClearResponse' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + + /superuser/search/system/cache/rebuild: + post: + tags: + - Search + summary: Queue system search cache rebuild + description: Queues a cache rebuild request and clears active query/intent cache namespaces immediately. + operationId: rebuildSystemSearchCache + requestBody: + required: false + content: + application/json: + schema: + $ref: '#/components/schemas/SystemSearchCacheRebuildRequest' + responses: + '200': + description: Cache rebuild queued successfully + content: + application/json: + schema: + $ref: '#/components/schemas/SystemSearchCacheRebuildResponse' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + # Configuration Endpoints /economic/config: get: @@ -9006,6 +9154,283 @@ components: type: integer description: HTTP status code + SystemSearchEntityType: + type: string + enum: + - objects + - module_config + - orders + - order_items + - customers + - employees + - subusers + - customer_discounts + - customer_fixed_prices + - departments + - permissions + - roles + - invoices + - vehicles + + SystemSearchRequest: + type: object + required: + - query + properties: + query: + type: string + description: Free-text query to search for. + include_types: + type: array + items: + $ref: '#/components/schemas/SystemSearchEntityType' + description: Limit search to these entity types. + exclude_types: + type: array + items: + $ref: '#/components/schemas/SystemSearchEntityType' + description: Exclude these entity types from search. + include_associations: + type: boolean + default: true + description: Include associated records for matched core entities. + debug_intent: + type: boolean + default: false + description: Include parser diagnostics in `meta.intent_parser`. + limit: + type: integer + minimum: 1 + maximum: 200 + default: 50 + offset: + type: integer + minimum: 0 + default: 0 + + SystemSearchResult: + type: object + properties: + entity_type: + $ref: '#/components/schemas/SystemSearchEntityType' + entity_id: + type: string + title: + type: string + description: + type: string + customer_number: + type: integer + nullable: true + department_id: + type: integer + nullable: true + score: + type: integer + association_reason: + type: string + nullable: true + payload: + type: object + additionalProperties: true + required: + - entity_type + - entity_id + - title + - score + + SystemSearchIntentParserMeta: + type: object + properties: + invoked: + type: boolean + source: + type: string + enum: [cache, openai, none] + status: + type: string + confidence: + type: number + minimum: 0 + maximum: 1 + expanded_terms: + type: array + items: + type: string + entity_hints: + type: array + items: + $ref: '#/components/schemas/SystemSearchEntityType' + fallback_reason: + type: string + nullable: true + required: + - invoked + - source + - status + - confidence + - expanded_terms + - entity_hints + + SystemSearchMeta: + type: object + properties: + query: + type: string + limit: + type: integer + offset: + type: integer + total: + type: integer + allowed_types: + type: array + items: + $ref: '#/components/schemas/SystemSearchEntityType' + cache: + type: object + properties: + hit: + type: boolean + required: [hit] + intent_parser: + $ref: '#/components/schemas/SystemSearchIntentParserMeta' + required: + - query + - limit + - offset + - total + - allowed_types + - cache + + SystemSearchPayload: + type: object + properties: + results: + type: array + items: + $ref: '#/components/schemas/SystemSearchResult' + grouped_results: + type: object + additionalProperties: + type: array + items: + $ref: '#/components/schemas/SystemSearchResult' + meta: + $ref: '#/components/schemas/SystemSearchMeta' + required: + - results + - grouped_results + - meta + + SystemSearchResponse: + type: object + properties: + success: + type: boolean + data: + $ref: '#/components/schemas/SystemSearchPayload' + meta: + type: object + additionalProperties: true + includes: + type: object + additionalProperties: true + required: + - success + - data + - meta + - includes + + SystemSearchCacheClearResponse: + type: object + properties: + success: + type: boolean + data: + type: object + properties: + message: + type: string + query_cache_cleared: + type: boolean + intent_cache_cleared: + type: boolean + required: + - message + - query_cache_cleared + - intent_cache_cleared + meta: + type: object + additionalProperties: true + includes: + type: object + additionalProperties: true + required: + - success + - data + - meta + - includes + + SystemSearchCacheRebuildRequest: + type: object + properties: + scope: + type: string + enum: [all, types, dirty] + default: all + types: + type: array + items: + $ref: '#/components/schemas/SystemSearchEntityType' + + SystemSearchCacheRebuildResponse: + type: object + properties: + success: + type: boolean + data: + type: object + properties: + message: + type: string + request: + type: object + properties: + scope: + type: string + enum: [all, types, dirty] + types: + type: array + items: + $ref: '#/components/schemas/SystemSearchEntityType' + requested_at: + type: integer + required: + - scope + - types + - requested_at + query_cache_cleared: + type: boolean + intent_cache_cleared: + type: boolean + required: + - message + - request + - query_cache_cleared + - intent_cache_cleared + meta: + type: object + additionalProperties: true + includes: + type: object + additionalProperties: true + required: + - success + - data + - meta + - includes + ModuleConfigValue: oneOf: - type: string diff --git a/services/nginx/app/classes/object_property.php b/services/nginx/app/classes/object_property.php index cae4ca53..3f858d44 100644 --- a/services/nginx/app/classes/object_property.php +++ b/services/nginx/app/classes/object_property.php @@ -155,6 +155,10 @@ class object_property if (defined('redis')) { redis->delete($this->getCacheKey()); } + try { + system_search_cache::markDirtyTable($this->table); + } catch (\Throwable) { + } } /** @@ -184,5 +188,9 @@ class object_property if (defined('redis')) { redis->delete($this->getCacheKey()); } + try { + system_search_cache::markDirtyTable($this->table); + } catch (\Throwable) { + } } -} \ No newline at end of file +} diff --git a/services/nginx/app/classes/system_search_cache.php b/services/nginx/app/classes/system_search_cache.php new file mode 100644 index 00000000..c0f8cd1c --- /dev/null +++ b/services/nginx/app/classes/system_search_cache.php @@ -0,0 +1,234 @@ + in_array($scope, ['all', 'types', 'dirty'], true) ? $scope : 'all', + 'types' => array_values(array_unique(array_filter(array_map('strval', $types)))), + 'requested_at' => time(), + ]; + self::redisSet(self::REBUILD_REQUEST_KEY, json_encode($payload, JSON_UNESCAPED_UNICODE)); + self::redisExpire(self::REBUILD_REQUEST_KEY, 86400); + return $payload; + } + + public static function consumeRebuildRequest(): ?array + { + $raw = self::redisGet(self::REBUILD_REQUEST_KEY); + if ($raw === null) { + return null; + } + self::redisDelete(self::REBUILD_REQUEST_KEY); + $decoded = json_decode($raw, true); + return is_array($decoded) ? $decoded : null; + } + + private static function clearPattern(string $pattern): void + { + try { + $client = self::redisClient(); + if ($client === null) { + return; + } + $client->clear_keys($pattern); + } catch (Throwable) { + // Cache clear must never break request flow. + } + } + + private static function redisSetEx(string $key, string $value, int $ttl): void + { + try { + $client = self::redisClient(); + if ($client === null) { + return; + } + $client->setEx($key, $value, $ttl); + } catch (Throwable) { + } + } + + private static function redisSet(string $key, string $value): void + { + try { + $client = self::redisClient(); + if ($client === null) { + return; + } + $client->set($key, $value); + } catch (Throwable) { + } + } + + private static function redisGet(string $key): ?string + { + try { + $client = self::redisClient(); + if ($client === null) { + return null; + } + $value = $client->get($key); + return is_string($value) ? $value : null; + } catch (Throwable) { + return null; + } + } + + private static function redisDelete(string $key): void + { + try { + $client = self::redisClient(); + if ($client === null) { + return; + } + $client->delete($key); + } catch (Throwable) { + } + } + + private static function redisSetArray(string $key, array $value): void + { + try { + $client = self::redisClient(); + if ($client === null) { + return; + } + $client->set_array($key, $value); + } catch (Throwable) { + } + } + + private static function redisGetArray(string $key): array + { + try { + $client = self::redisClient(); + if ($client === null) { + return []; + } + $value = $client->get_array($key); + return is_array($value) ? $value : []; + } catch (Throwable) { + return []; + } + } + + private static function redisExpire(string $key, int $ttl): void + { + try { + $client = self::redisClient(); + if ($client === null) { + return; + } + $client->expire($key, $ttl); + } catch (Throwable) { + } + } + + private static function redisClient(): ?object + { + if (self::$adapter !== null) { + return self::$adapter; + } + if (!defined('redis')) { + return null; + } + $client = constant('redis'); + return is_object($client) ? $client : null; + } +} diff --git a/services/nginx/app/classes/system_search_openai_intent_parser.php b/services/nginx/app/classes/system_search_openai_intent_parser.php new file mode 100644 index 00000000..5d6d347f --- /dev/null +++ b/services/nginx/app/classes/system_search_openai_intent_parser.php @@ -0,0 +1,247 @@ +transport = $transport; + $this->forcedEnabled = $forcedEnabled; + $this->forcedApiKey = $forcedApiKey; + } + + public function parse(string $query, array $allowedEntityTypes, array $taxonomy = []): array + { + $query = trim($query); + if ($query === '') { + return $this->failed('empty_query', 'none'); + } + + [$enabled, $apiKey] = $this->resolveOpenAISettings(); + if (!$enabled) { + return $this->failed('openai_disabled', 'none'); + } + if (empty($apiKey)) { + return $this->failed('openai_missing_key', 'none'); + } + + $redactedQuery = self::redactSensitiveQuery($query); + $payload = $this->buildPayload($redactedQuery, $allowedEntityTypes, $taxonomy); + $cacheHash = md5(json_encode([ + 'q' => $redactedQuery, + 'types' => $allowedEntityTypes, + 'taxonomy' => $taxonomy, + 'v' => 1, + ], JSON_UNESCAPED_UNICODE)); + + $cached = system_search_cache::getIntent($cacheHash); + if (is_array($cached) && isset($cached['success'])) { + $cached['source'] = 'cache'; + return $this->normalizeResult($cached); + } + + try { + $raw = $this->sendRequest($payload, $apiKey); + $parsed = $this->parseResponse($raw); + $parsed['source'] = 'openai'; + system_search_cache::setIntent($cacheHash, $parsed, 3600); + return $this->normalizeResult($parsed); + } catch (Throwable $e) { + return $this->failed($e->getMessage(), 'openai'); + } + } + + public static function redactSensitiveQuery(string $query): string + { + $query = preg_replace('/[A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,}/i', '[email]', $query) ?? $query; + $query = preg_replace('/\b\d{8}\b/', '[cvr]', $query) ?? $query; + $query = preg_replace('/\+?\d[\d\s\-]{6,}\d/', '[phone]', $query) ?? $query; + return $query; + } + + private function resolveOpenAISettings(): array + { + if ($this->forcedEnabled !== null) { + return [(bool)$this->forcedEnabled, (string)($this->forcedApiKey ?? '')]; + } + + try { + $openai = new openai(); + $enabled = (bool)$openai->config->enabled->getVariableValue(); + $apiKey = (string)$openai->config->api_key->getVariableValue(); + return [$enabled, $apiKey]; + } catch (Throwable) { + return [false, '']; + } + } + + private function buildPayload(string $query, array $allowedEntityTypes, array $taxonomy): array + { + $taxonomyText = json_encode([ + 'allowed_entity_types' => array_values($allowedEntityTypes), + 'taxonomy' => $taxonomy, + ], JSON_UNESCAPED_UNICODE); + + $prompt = "You parse user search intent into strict JSON.\n" + . "Rules:\n" + . "- Keep output concise and valid JSON only.\n" + . "- Do not invent entity types not listed in allowed_entity_types.\n" + . "- aliases should contain user-friendly alternative terms.\n" + . "- confidence must be between 0 and 1.\n" + . "- association_hint should be true if related records likely needed.\n\n" + . "Context:\n" + . $taxonomyText . "\n\n" + . "User query:\n" + . $query; + + return [ + 'model' => $this->model, + 'temperature' => $this->temperature, + 'input' => [ + [ + 'role' => 'user', + 'content' => [ + ['type' => 'input_text', 'text' => $prompt], + ], + ], + ], + 'text' => [ + 'format' => [ + 'type' => 'json_schema', + 'name' => 'system_search_intent', + 'schema' => [ + 'type' => 'object', + 'properties' => [ + 'success' => ['type' => 'boolean'], + 'normalized_query' => ['type' => 'string'], + 'aliases' => [ + 'type' => 'array', + 'items' => ['type' => 'string'], + ], + 'entity_hints' => [ + 'type' => 'array', + 'items' => ['type' => 'string'], + ], + 'confidence' => [ + 'type' => 'number', + 'minimum' => 0, + 'maximum' => 1, + ], + 'association_hint' => ['type' => 'boolean'], + 'fallback_reason' => ['type' => ['string', 'null']], + ], + 'required' => [ + 'success', + 'normalized_query', + 'aliases', + 'entity_hints', + 'confidence', + 'association_hint', + 'fallback_reason', + ], + 'additionalProperties' => false, + ], + 'strict' => true, + ], + ], + ]; + } + + private function sendRequest(array $payload, string $apiKey): array + { + if ($this->transport !== null) { + $result = call_user_func($this->transport, $payload, $apiKey); + if (!is_array($result)) { + throw new Exception('Transport returned invalid payload'); + } + return $result; + } + + $curl = curl_init($this->apiUrl); + curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curl, CURLOPT_POST, true); + curl_setopt($curl, CURLOPT_TIMEOUT, $this->timeoutSeconds); + curl_setopt($curl, CURLOPT_HTTPHEADER, [ + 'Content-Type: application/json', + 'Authorization: Bearer ' . $apiKey, + ]); + curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($payload, JSON_UNESCAPED_UNICODE)); + $raw = curl_exec($curl); + if ($raw === false) { + $error = curl_error($curl); + curl_close($curl); + throw new Exception('cURL error: ' . $error); + } + $status = curl_getinfo($curl, CURLINFO_HTTP_CODE); + curl_close($curl); + $decoded = json_decode($raw, true); + if (!is_array($decoded)) { + throw new Exception('Invalid JSON from OpenAI'); + } + if ($status >= 400) { + $message = $decoded['error']['message'] ?? ('OpenAI HTTP ' . $status); + throw new Exception($message); + } + return $decoded; + } + + private function parseResponse(array $response): array + { + $text = $response['output'][0]['content'][0]['text'] ?? null; + if (!is_string($text) || $text === '') { + throw new Exception('Invalid response format (missing output text)'); + } + $decoded = json_decode($text, true); + if (!is_array($decoded)) { + throw new Exception('Invalid intent JSON'); + } + return $decoded; + } + + private function normalizeResult(array $result): array + { + $entityHints = array_values(array_unique(array_filter(array_map('strval', (array)($result['entity_hints'] ?? []))))); + $aliases = array_values(array_unique(array_filter(array_map('strval', (array)($result['aliases'] ?? []))))); + return [ + 'success' => (bool)($result['success'] ?? false), + 'normalized_query' => (string)($result['normalized_query'] ?? ''), + 'aliases' => $aliases, + 'entity_hints' => $entityHints, + 'confidence' => max(0.0, min(1.0, (float)($result['confidence'] ?? 0.0))), + 'association_hint' => (bool)($result['association_hint'] ?? false), + 'fallback_reason' => isset($result['fallback_reason']) ? (string)$result['fallback_reason'] : null, + 'source' => (string)($result['source'] ?? 'openai'), + ]; + } + + private function failed(string $reason, string $source): array + { + return [ + 'success' => false, + 'normalized_query' => '', + 'aliases' => [], + 'entity_hints' => [], + 'confidence' => 0.0, + 'association_hint' => false, + 'fallback_reason' => $reason, + 'source' => $source, + ]; + } +} diff --git a/services/nginx/app/classes/system_search_service.php b/services/nginx/app/classes/system_search_service.php new file mode 100644 index 00000000..7347e06a --- /dev/null +++ b/services/nginx/app/classes/system_search_service.php @@ -0,0 +1,1062 @@ +intentParser = $intentParser ?? new system_search_openai_intent_parser(); + } + + public function search(array $options): array + { + $query = trim((string)($options['query'] ?? '')); + $includeTypes = $this->normalizeTypes((array)($options['include_types'] ?? [])); + $excludeTypes = $this->normalizeTypes((array)($options['exclude_types'] ?? [])); + $allowedTypes = $this->normalizeTypes((array)($options['allowed_types'] ?? [])); + $ownOnlyTypes = $this->normalizeTypes((array)($options['own_only_types'] ?? [])); + $ownCustomerNumber = isset($options['own_customer_number']) ? (int)$options['own_customer_number'] : null; + $permissionsCatalogAll = (array)($options['permissions_catalog_all'] ?? []); + $permissionsCatalogOwn = (array)($options['permissions_catalog_own'] ?? []); + $moduleConfigVisibility = (array)($options['module_config_visibility'] ?? []); + $includeAssociations = (bool)($options['include_associations'] ?? true); + $debugIntent = (bool)($options['debug_intent'] ?? false); + + $limit = (int)($options['limit'] ?? 50); + $offset = (int)($options['offset'] ?? 0); + if ($limit < 1) { + $limit = 50; + } + if ($limit > 200) { + $limit = 200; + } + if ($offset < 0) { + $offset = 0; + } + + $allTypes = $this->allEntityTypes(); + $activeTypes = empty($includeTypes) ? $allTypes : array_values(array_intersect($allTypes, $includeTypes)); + if (!empty($excludeTypes)) { + $activeTypes = array_values(array_diff($activeTypes, $excludeTypes)); + } + $activeTypes = array_values(array_intersect($activeTypes, $allowedTypes)); + + $baseMeta = [ + 'query' => $query, + 'limit' => $limit, + 'offset' => $offset, + 'allowed_types' => $activeTypes, + 'cache' => ['hit' => false], + ]; + + if ($query === '' || empty($activeTypes)) { + return [ + 'results' => [], + 'grouped_results' => $this->groupResultsByType([]), + 'meta' => [ + ...$baseMeta, + 'total' => 0, + ], + ]; + } + + $queryCacheHash = md5(json_encode([ + 'q' => $query, + 'include' => $includeTypes, + 'exclude' => $excludeTypes, + 'active' => $activeTypes, + 'limit' => $limit, + 'offset' => $offset, + 'own' => $ownCustomerNumber, + 'own_only' => $ownOnlyTypes, + 'assoc' => $includeAssociations, + 'dbg' => $debugIntent, + 'v' => 2, + ], JSON_UNESCAPED_UNICODE)); + + $cached = system_search_cache::getQuery($queryCacheHash); + if (is_array($cached) && isset($cached['results'], $cached['grouped_results'], $cached['meta'])) { + $cached['meta']['cache'] = ['hit' => true]; + return $cached; + } + + $terms = $this->tokenize($query); + $entityBoost = []; + $initialResults = $this->executeLexicalSearch( + $activeTypes, + $terms, + $entityBoost, + $ownOnlyTypes, + $ownCustomerNumber, + $permissionsCatalogAll, + $permissionsCatalogOwn, + $moduleConfigVisibility + ); + + $intentMeta = [ + 'invoked' => false, + 'source' => 'none', + 'status' => 'skipped', + 'confidence' => 0.0, + 'expanded_terms' => $terms, + 'entity_hints' => [], + 'fallback_reason' => null, + ]; + + if ($this->shouldInvokeIntentParser($initialResults) && !empty($terms)) { + $intentMeta['invoked'] = true; + $taxonomy = $this->taxonomy($activeTypes); + $intent = $this->intentParser->parse($query, $activeTypes, $taxonomy); + $intentMeta['source'] = (string)($intent['source'] ?? 'none'); + $intentMeta['confidence'] = (float)($intent['confidence'] ?? 0.0); + $intentMeta['fallback_reason'] = $intent['fallback_reason'] ?? null; + $intentMeta['entity_hints'] = (array)($intent['entity_hints'] ?? []); + + if (!empty($intent['success'])) { + $intentMeta['status'] = 'ok'; + $expandedTerms = array_values(array_unique([ + ...$terms, + ...$this->tokenize((string)($intent['normalized_query'] ?? '')), + ...$this->tokenize(implode(' ', (array)($intent['aliases'] ?? []))), + ])); + $intentMeta['expanded_terms'] = $expandedTerms; + + $boostedTypes = array_values(array_intersect($activeTypes, (array)($intent['entity_hints'] ?? []))); + foreach ($boostedTypes as $boostedType) { + $entityBoost[$boostedType] = 25; + } + + $initialResults = $this->executeLexicalSearch( + $activeTypes, + $expandedTerms, + $entityBoost, + $ownOnlyTypes, + $ownCustomerNumber, + $permissionsCatalogAll, + $permissionsCatalogOwn, + $moduleConfigVisibility + ); + } else { + $intentMeta['status'] = 'fallback'; + } + } + + if ($includeAssociations) { + $customerNumbers = []; + foreach ($initialResults as $result) { + if ($result['entity_type'] === 'customers' && isset($result['customer_number'])) { + $customerNumbers[] = (int)$result['customer_number']; + } + } + $customerNumbers = array_values(array_unique(array_filter($customerNumbers))); + if (!empty($customerNumbers)) { + $associationTypes = array_values(array_intersect( + $activeTypes, + ['orders', 'order_items', 'invoices', 'vehicles', 'customer_discounts', 'customer_fixed_prices'] + )); + foreach ($customerNumbers as $customerNumber) { + $associated = $this->executeLexicalSearch( + $associationTypes, + [(string)$customerNumber], + [], + $ownOnlyTypes, + $ownCustomerNumber, + $permissionsCatalogAll, + $permissionsCatalogOwn, + $moduleConfigVisibility, + [$customerNumber] + ); + foreach ($associated as &$item) { + if (!isset($item['association_reason'])) { + $item['association_reason'] = 'customer:' . $customerNumber; + } + $item['score'] = max((int)$item['score'], 35); + } + $initialResults = $this->mergeResults($initialResults, $associated); + } + } + } + + usort($initialResults, function (array $a, array $b): int { + if ($a['score'] === $b['score']) { + return strcmp((string)$a['entity_type'] . ':' . (string)$a['entity_id'], (string)$b['entity_type'] . ':' . (string)$b['entity_id']); + } + return $b['score'] <=> $a['score']; + }); + + $total = count($initialResults); + $paged = array_slice($initialResults, $offset, $limit); + $grouped = $this->groupResultsByType($paged); + + $meta = [ + ...$baseMeta, + 'total' => $total, + ]; + if ($debugIntent) { + $meta['intent_parser'] = $intentMeta; + } + + $payload = [ + 'results' => $paged, + 'grouped_results' => $grouped, + 'meta' => $meta, + ]; + system_search_cache::setQuery($queryCacheHash, $payload, 120); + + return $payload; + } + + protected function shouldInvokeIntentParser(array $results): bool + { + if (count($results) < $this->lowConfidenceResultThreshold) { + return true; + } + $topScore = (int)($results[0]['score'] ?? 0); + return $topScore < $this->lowConfidenceTopScoreThreshold; + } + + /** + * @param array $activeTypes + * @param array $terms + * @param array $entityBoost + * @param array $ownOnlyTypes + * @param int|null $ownCustomerNumber + * @param array $permissionsCatalogAll + * @param array $permissionsCatalogOwn + * @param array $moduleConfigVisibility + * @param array $forcedCustomerNumbers + * @return array> + */ + protected function executeLexicalSearch( + array $activeTypes, + array $terms, + array $entityBoost, + array $ownOnlyTypes, + ?int $ownCustomerNumber, + array $permissionsCatalogAll, + array $permissionsCatalogOwn, + array $moduleConfigVisibility, + array $forcedCustomerNumbers = [] + ): array { + $results = []; + foreach ($activeTypes as $entityType) { + $boost = (int)($entityBoost[$entityType] ?? 0); + $ownOnly = in_array($entityType, $ownOnlyTypes, true); + $rows = $this->searchEntity( + $entityType, + $terms, + $boost, + $ownOnly, + $ownCustomerNumber, + $permissionsCatalogAll, + $permissionsCatalogOwn, + $moduleConfigVisibility, + $forcedCustomerNumbers + ); + $results = $this->mergeResults($results, $rows); + } + return $results; + } + + /** + * @param array $terms + * @param array $permissionsCatalogAll + * @param array $permissionsCatalogOwn + * @param array $moduleConfigVisibility + * @param array $forcedCustomerNumbers + * @return array> + */ + private function searchEntity( + string $entityType, + array $terms, + int $entityBoost, + bool $ownOnly, + ?int $ownCustomerNumber, + array $permissionsCatalogAll, + array $permissionsCatalogOwn, + array $moduleConfigVisibility, + array $forcedCustomerNumbers + ): array { + return match ($entityType) { + 'customers' => $this->searchCustomers($terms, $entityBoost, $ownOnly, $ownCustomerNumber, $forcedCustomerNumbers), + 'employees' => $this->searchEmployees($terms, $entityBoost, $ownOnly, $ownCustomerNumber), + 'orders' => $this->searchOrders($terms, $entityBoost, $ownOnly, $ownCustomerNumber, $forcedCustomerNumbers), + 'order_items' => $this->searchOrderItems($terms, $entityBoost, $ownOnly, $ownCustomerNumber, $forcedCustomerNumbers), + 'invoices' => $this->searchInvoices($terms, $entityBoost, $ownOnly, $ownCustomerNumber, $forcedCustomerNumbers), + 'vehicles' => $this->searchVehicles($terms, $entityBoost, $ownOnly, $ownCustomerNumber, $forcedCustomerNumbers), + 'subusers' => $this->searchSubusers($terms, $entityBoost, $ownOnly, $ownCustomerNumber), + 'customer_discounts' => $this->searchCustomerDiscounts($terms, $entityBoost, $ownOnly, $ownCustomerNumber, $forcedCustomerNumbers), + 'customer_fixed_prices' => $this->searchCustomerFixedPrices($terms, $entityBoost, $ownOnly, $ownCustomerNumber, $forcedCustomerNumbers), + 'departments' => $this->searchDepartments($terms, $entityBoost), + 'roles' => $this->searchRoles($terms, $entityBoost), + 'permissions' => $this->searchPermissions($terms, $entityBoost, $ownOnly, $permissionsCatalogAll, $permissionsCatalogOwn), + 'module_config' => $this->searchModuleConfig($terms, $entityBoost, $moduleConfigVisibility), + 'objects' => $this->searchObjects($terms, $entityBoost, $ownOnly, $ownCustomerNumber), + default => [], + }; + } + + private function searchCustomers(array $terms, int $entityBoost, bool $ownOnly, ?int $ownCustomerNumber, array $forcedCustomerNumbers): array + { + $customerNumbers = !empty($forcedCustomerNumbers) ? $forcedCustomerNumbers : ($ownOnly && $ownCustomerNumber ? [$ownCustomerNumber] : []); + $rows = $this->searchTable( + 'users', + ['id', 'customer_number', 'display_name', 'email', 'phone'], + ['id', 'customer_number', 'display_name', 'email', 'phone'], + $terms, + $customerNumbers, + 'customer_number' + ); + + return array_map(function (array $row) use ($terms, $entityBoost) { + return [ + 'entity_type' => 'customers', + 'entity_id' => (string)$row['id'], + 'title' => (string)($row['display_name'] ?: ('Customer #' . $row['customer_number'])), + 'description' => (string)($row['email'] ?? ''), + 'customer_number' => isset($row['customer_number']) ? (int)$row['customer_number'] : null, + 'score' => $this->scoreRow($row, ['customer_number', 'display_name', 'email', 'phone'], $terms) + $entityBoost, + 'payload' => [ + 'id' => (int)$row['id'], + 'customer_number' => isset($row['customer_number']) ? (int)$row['customer_number'] : null, + 'display_name' => $row['display_name'] ?? null, + 'email' => $row['email'] ?? null, + 'phone' => $row['phone'] ?? null, + ], + ]; + }, $rows); + } + + private function searchEmployees(array $terms, int $entityBoost, bool $ownOnly, ?int $ownCustomerNumber): array + { + $rows = $this->searchTableWithJoin( + 'users', + 'users u INNER JOIN groups_permissions gp ON gp.group_id = u.group_id', + ['u.id', 'u.customer_number', 'u.display_name', 'u.email', 'u.phone'], + ['u.id', 'u.customer_number', 'u.display_name', 'u.email', 'u.phone'], + $terms, + "gp.permission = 'employee_public_data'" . (($ownOnly && $ownCustomerNumber) ? (' AND u.customer_number = ' . (int)$ownCustomerNumber) : '') + ); + + return array_map(function (array $row) use ($terms, $entityBoost) { + return [ + 'entity_type' => 'employees', + 'entity_id' => (string)$row['id'], + 'title' => (string)($row['display_name'] ?: ('Employee #' . $row['id'])), + 'description' => (string)($row['email'] ?? ''), + 'customer_number' => isset($row['customer_number']) ? (int)$row['customer_number'] : null, + 'score' => $this->scoreRow($row, ['display_name', 'email', 'phone', 'customer_number'], $terms) + $entityBoost, + 'payload' => [ + 'id' => (int)$row['id'], + 'display_name' => $row['display_name'] ?? null, + 'email' => $row['email'] ?? null, + 'phone' => $row['phone'] ?? null, + ], + ]; + }, $rows); + } + + private function searchOrders(array $terms, int $entityBoost, bool $ownOnly, ?int $ownCustomerNumber, array $forcedCustomerNumbers): array + { + $customerNumbers = !empty($forcedCustomerNumbers) ? $forcedCustomerNumbers : ($ownOnly && $ownCustomerNumber ? [$ownCustomerNumber] : []); + $rows = $this->searchTable( + 'orders', + ['id', 'customer_id', 'reference', 'notes', 'reg_1', 'reg_2', 'reg_3', 'department_id', 'po', 'deleted_at'], + ['id', 'customer_id', 'reference', 'notes', 'reg_1', 'reg_2', 'reg_3', 'po'], + $terms, + $customerNumbers, + 'customer_id', + ['deleted_at' => null] + ); + + return array_map(function (array $row) use ($terms, $entityBoost) { + return [ + 'entity_type' => 'orders', + 'entity_id' => (string)$row['id'], + 'title' => 'Order #' . (string)$row['id'], + 'description' => (string)($row['reference'] ?? ''), + 'customer_number' => isset($row['customer_id']) ? (int)$row['customer_id'] : null, + 'department_id' => isset($row['department_id']) ? (int)$row['department_id'] : null, + 'score' => $this->scoreRow($row, ['id', 'customer_id', 'reference', 'notes', 'reg_1', 'reg_2', 'reg_3', 'po'], $terms) + $entityBoost, + 'payload' => [ + 'id' => (int)$row['id'], + 'customer_id' => isset($row['customer_id']) ? (int)$row['customer_id'] : null, + 'reference' => $row['reference'] ?? null, + 'notes' => $row['notes'] ?? null, + 'reg_1' => $row['reg_1'] ?? null, + ], + ]; + }, $rows); + } + + private function searchOrderItems(array $terms, int $entityBoost, bool $ownOnly, ?int $ownCustomerNumber, array $forcedCustomerNumbers): array + { + $customerClause = ''; + if (!empty($forcedCustomerNumbers)) { + $customerClause = ' AND o.customer_id IN (' . implode(',', array_map('intval', $forcedCustomerNumbers)) . ')'; + } elseif ($ownOnly && $ownCustomerNumber) { + $customerClause = ' AND o.customer_id = ' . (int)$ownCustomerNumber; + } + $baseWhere = 'o.id = oi.order_id' . $customerClause . ' AND o.deleted_at IS NULL'; + $rows = $this->searchTableWithJoin( + 'order_items', + 'order_items oi INNER JOIN orders o ON o.id = oi.order_id', + ['oi.id', 'oi.order_id', 'oi.product_id', 'oi.reference', 'oi.notes', 'o.customer_id AS customer_number'], + ['oi.id', 'oi.order_id', 'oi.product_id', 'oi.reference', 'oi.notes', 'o.customer_id'], + $terms, + $baseWhere + ); + + return array_map(function (array $row) use ($terms, $entityBoost) { + return [ + 'entity_type' => 'order_items', + 'entity_id' => (string)$row['id'], + 'title' => 'Order item #' . (string)$row['id'], + 'description' => (string)($row['reference'] ?? ''), + 'customer_number' => isset($row['customer_number']) ? (int)$row['customer_number'] : null, + 'score' => $this->scoreRow($row, ['id', 'order_id', 'product_id', 'reference', 'notes', 'customer_number'], $terms) + $entityBoost, + 'payload' => [ + 'id' => (int)$row['id'], + 'order_id' => isset($row['order_id']) ? (int)$row['order_id'] : null, + 'product_id' => isset($row['product_id']) ? (int)$row['product_id'] : null, + 'reference' => $row['reference'] ?? null, + ], + ]; + }, $rows); + } + + private function searchInvoices(array $terms, int $entityBoost, bool $ownOnly, ?int $ownCustomerNumber, array $forcedCustomerNumbers): array + { + $customerNumbers = !empty($forcedCustomerNumbers) ? $forcedCustomerNumbers : ($ownOnly && $ownCustomerNumber ? [$ownCustomerNumber] : []); + $rows = $this->searchTable( + 'collected_order_invoices', + ['id', 'customer_number', 'name', 'notes', 'external_id', 'booked_invoice_id', 'po_number', 'deleted_at'], + ['id', 'customer_number', 'name', 'notes', 'external_id', 'booked_invoice_id', 'po_number'], + $terms, + $customerNumbers, + 'customer_number', + ['deleted_at' => null] + ); + + return array_map(function (array $row) use ($terms, $entityBoost) { + return [ + 'entity_type' => 'invoices', + 'entity_id' => (string)$row['id'], + 'title' => (string)($row['name'] ?: ('Invoice collection #' . $row['id'])), + 'description' => (string)($row['external_id'] ?? ''), + 'customer_number' => isset($row['customer_number']) ? (int)$row['customer_number'] : null, + 'score' => $this->scoreRow($row, ['id', 'customer_number', 'name', 'notes', 'external_id', 'booked_invoice_id', 'po_number'], $terms) + $entityBoost, + 'payload' => [ + 'id' => (int)$row['id'], + 'customer_number' => isset($row['customer_number']) ? (int)$row['customer_number'] : null, + 'name' => $row['name'] ?? null, + 'external_id' => $row['external_id'] ?? null, + ], + ]; + }, $rows); + } + + private function searchVehicles(array $terms, int $entityBoost, bool $ownOnly, ?int $ownCustomerNumber, array $forcedCustomerNumbers): array + { + $customerNumbers = !empty($forcedCustomerNumbers) ? $forcedCustomerNumbers : ($ownOnly && $ownCustomerNumber ? [$ownCustomerNumber] : []); + $rows = $this->searchTable( + 'customer_vehicles', + ['id', 'customer_id', 'reg', 'reference', 'type', 'deleted_at'], + ['id', 'customer_id', 'reg', 'reference', 'type'], + $terms, + $customerNumbers, + 'customer_id', + ['deleted_at' => null] + ); + + return array_map(function (array $row) use ($terms, $entityBoost) { + return [ + 'entity_type' => 'vehicles', + 'entity_id' => (string)$row['id'], + 'title' => (string)($row['reg'] ?: ('Vehicle #' . $row['id'])), + 'description' => (string)($row['reference'] ?? ''), + 'customer_number' => isset($row['customer_id']) ? (int)$row['customer_id'] : null, + 'score' => $this->scoreRow($row, ['id', 'customer_id', 'reg', 'reference', 'type'], $terms) + $entityBoost, + 'payload' => [ + 'id' => (int)$row['id'], + 'customer_id' => isset($row['customer_id']) ? (int)$row['customer_id'] : null, + 'reg' => $row['reg'] ?? null, + 'reference' => $row['reference'] ?? null, + ], + ]; + }, $rows); + } + + private function searchSubusers(array $terms, int $entityBoost, bool $ownOnly, ?int $ownCustomerNumber): array + { + $where = '1=1'; + if ($ownOnly && $ownCustomerNumber) { + $where .= ' AND sg.billing_customer_number = ' . (int)$ownCustomerNumber . ' AND sg.deleted_at IS NULL'; + } + $rows = $this->searchTableWithJoin( + 'subusers', + 'subusers s LEFT JOIN subuser_grants sg ON sg.subuser = s.id', + ['s.id', 's.username', 's.name', 's.email', 's.phone_country_code', 's.phone'], + ['s.id', 's.username', 's.name', 's.email', 's.phone'], + $terms, + $where + ); + + return array_map(function (array $row) use ($terms, $entityBoost) { + return [ + 'entity_type' => 'subusers', + 'entity_id' => (string)$row['id'], + 'title' => (string)($row['name'] ?: ($row['username'] ?? ('Subuser #' . $row['id']))), + 'description' => (string)($row['email'] ?? ''), + 'score' => $this->scoreRow($row, ['id', 'username', 'name', 'email', 'phone'], $terms) + $entityBoost, + 'payload' => [ + 'id' => (int)$row['id'], + 'username' => $row['username'] ?? null, + 'name' => $row['name'] ?? null, + 'email' => $row['email'] ?? null, + ], + ]; + }, $rows); + } + + private function searchCustomerDiscounts(array $terms, int $entityBoost, bool $ownOnly, ?int $ownCustomerNumber, array $forcedCustomerNumbers): array + { + $customerFilter = ''; + if (!empty($forcedCustomerNumbers)) { + $customerFilter = ' AND u.customer_number IN (' . implode(',', array_map('intval', $forcedCustomerNumbers)) . ')'; + } elseif ($ownOnly && $ownCustomerNumber) { + $customerFilter = ' AND u.customer_number = ' . (int)$ownCustomerNumber; + } + + $rows = $this->searchTableWithJoin( + 'price_overrides', + 'price_overrides po INNER JOIN users u ON u.id = po.user_id', + ['po.id', 'po.user_id', 'po.is_category', 'po.product_or_category_id', 'po.percentage', 'u.customer_number', 'u.display_name'], + ['po.id', 'po.user_id', 'po.product_or_category_id', 'po.percentage', 'u.customer_number', 'u.display_name'], + $terms, + '1=1' . $customerFilter + ); + + return array_map(function (array $row) use ($terms, $entityBoost) { + return [ + 'entity_type' => 'customer_discounts', + 'entity_id' => (string)$row['id'], + 'title' => 'Discount #' . (string)$row['id'], + 'description' => (string)('Customer ' . ($row['customer_number'] ?? '') . ' / ' . ($row['display_name'] ?? '')), + 'customer_number' => isset($row['customer_number']) ? (int)$row['customer_number'] : null, + 'score' => $this->scoreRow($row, ['id', 'customer_number', 'display_name', 'product_or_category_id', 'percentage', 'user_id'], $terms) + $entityBoost, + 'payload' => [ + 'id' => (int)$row['id'], + 'customer_number' => isset($row['customer_number']) ? (int)$row['customer_number'] : null, + 'product_or_category_id' => $row['product_or_category_id'] ?? null, + 'percentage' => isset($row['percentage']) ? (int)$row['percentage'] : null, + ], + ]; + }, $rows); + } + + private function searchCustomerFixedPrices(array $terms, int $entityBoost, bool $ownOnly, ?int $ownCustomerNumber, array $forcedCustomerNumbers): array + { + $customerNumbers = !empty($forcedCustomerNumbers) ? $forcedCustomerNumbers : ($ownOnly && $ownCustomerNumber ? [$ownCustomerNumber] : []); + $rows = $this->searchTable( + 'customer_fixed_pricing', + ['id', 'customer_number', 'price', 'description'], + ['id', 'customer_number', 'price', 'description'], + $terms, + $customerNumbers, + 'customer_number' + ); + + return array_map(function (array $row) use ($terms, $entityBoost) { + return [ + 'entity_type' => 'customer_fixed_prices', + 'entity_id' => (string)$row['id'], + 'title' => 'Fixed pricing #' . (string)$row['id'], + 'description' => (string)($row['description'] ?? ''), + 'customer_number' => isset($row['customer_number']) ? (int)$row['customer_number'] : null, + 'score' => $this->scoreRow($row, ['id', 'customer_number', 'price', 'description'], $terms) + $entityBoost, + 'payload' => [ + 'id' => (int)$row['id'], + 'customer_number' => isset($row['customer_number']) ? (int)$row['customer_number'] : null, + 'price' => isset($row['price']) ? (int)$row['price'] : null, + 'description' => $row['description'] ?? null, + ], + ]; + }, $rows); + } + + private function searchDepartments(array $terms, int $entityBoost): array + { + $rows = $this->searchTable( + 'departments', + ['id', 'name', 'address', 'zip', 'city'], + ['id', 'name', 'address', 'zip', 'city'], + $terms + ); + + return array_map(function (array $row) use ($terms, $entityBoost) { + return [ + 'entity_type' => 'departments', + 'entity_id' => (string)$row['id'], + 'title' => (string)($row['name'] ?: ('Department #' . $row['id'])), + 'description' => trim((string)(($row['address'] ?? '') . ' ' . ($row['city'] ?? ''))), + 'score' => $this->scoreRow($row, ['id', 'name', 'address', 'zip', 'city'], $terms) + $entityBoost, + 'payload' => [ + 'id' => (int)$row['id'], + 'name' => $row['name'] ?? null, + 'address' => $row['address'] ?? null, + 'zip' => $row['zip'] ?? null, + 'city' => $row['city'] ?? null, + ], + ]; + }, $rows); + } + + private function searchRoles(array $terms, int $entityBoost): array + { + $rows = $this->searchTable( + 'groups', + ['id', 'name', 'description'], + ['id', 'name', 'description'], + $terms + ); + + return array_map(function (array $row) use ($terms, $entityBoost) { + return [ + 'entity_type' => 'roles', + 'entity_id' => (string)$row['id'], + 'title' => (string)($row['name'] ?: ('Role #' . $row['id'])), + 'description' => (string)($row['description'] ?? ''), + 'score' => $this->scoreRow($row, ['id', 'name', 'description'], $terms) + $entityBoost, + 'payload' => [ + 'id' => (int)$row['id'], + 'name' => $row['name'] ?? null, + 'description' => $row['description'] ?? null, + ], + ]; + }, $rows); + } + + private function searchPermissions(array $terms, int $entityBoost, bool $ownOnly, array $permissionsCatalogAll, array $permissionsCatalogOwn): array + { + $rows = []; + if ($ownOnly) { + foreach ($permissionsCatalogOwn as $permission) { + $rows[] = ['permission' => (string)$permission, 'description' => (string)$permission]; + } + } else { + foreach ($permissionsCatalogAll as $permission => $description) { + $rows[] = ['permission' => (string)$permission, 'description' => (string)$description]; + } + } + + $filtered = []; + foreach ($rows as $row) { + $score = $this->scoreRow($row, ['permission', 'description'], $terms) + $entityBoost; + if ($score <= 0) { + continue; + } + $filtered[] = [ + 'entity_type' => 'permissions', + 'entity_id' => (string)$row['permission'], + 'title' => (string)$row['permission'], + 'description' => (string)$row['description'], + 'score' => $score, + 'payload' => $row, + ]; + } + return $filtered; + } + + private function searchModuleConfig(array $terms, int $entityBoost, array $moduleConfigVisibility): array + { + $rows = $this->searchTable( + 'module_config', + ['module', 'variable', 'type', 'value'], + ['module', 'variable', 'type'], + $terms + ); + + $filtered = []; + foreach ($rows as $row) { + $module = (string)($row['module'] ?? ''); + if ($module !== '' && isset($moduleConfigVisibility[$module]) && !$moduleConfigVisibility[$module]) { + continue; + } + $variable = (string)($row['variable'] ?? ''); + if ($this->looksSecretVariable($variable)) { + continue; + } + $score = $this->scoreRow($row, ['module', 'variable', 'type'], $terms) + $entityBoost; + if ($score <= 0) { + continue; + } + $filtered[] = [ + 'entity_type' => 'module_config', + 'entity_id' => $module . ':' . $variable, + 'title' => $module . '.' . $variable, + 'description' => (string)($row['type'] ?? ''), + 'score' => $score, + 'payload' => [ + 'module' => $module, + 'variable' => $variable, + 'type' => $row['type'] ?? null, + ], + ]; + } + return $filtered; + } + + private function searchObjects(array $terms, int $entityBoost, bool $ownOnly, ?int $ownCustomerNumber): array + { + if ($ownOnly && $ownCustomerNumber !== null) { + return []; + } + $rows = $this->searchTable( + 'object_attachments', + ['id', 'object_type', 'object_id', 'content', 'deleted_at'], + ['id', 'object_type', 'object_id', 'content'], + $terms, + [], + null, + ['deleted_at' => null] + ); + + return array_map(function (array $row) use ($terms, $entityBoost) { + return [ + 'entity_type' => 'objects', + 'entity_id' => (string)$row['id'], + 'title' => (string)($row['object_type'] ?? 'object_attachment') . '#' . (string)$row['object_id'], + 'description' => (string)$row['content'], + 'score' => $this->scoreRow($row, ['id', 'object_type', 'object_id', 'content'], $terms) + $entityBoost, + 'payload' => [ + 'id' => (int)$row['id'], + 'object_type' => $row['object_type'] ?? null, + 'object_id' => $row['object_id'] ?? null, + ], + ]; + }, $rows); + } + + /** + * Generic table search helper. + * + * @param array $candidateFields + * @param array $searchFields + * @param array $terms + * @param array $customerNumbers + * @param string|null $customerField + * @param array $fixedConditions + * @return array> + */ + private function searchTable( + string $table, + array $candidateFields, + array $searchFields, + array $terms, + array $customerNumbers = [], + ?string $customerField = null, + array $fixedConditions = [] + ): array { + global $db; + + if (!$this->tableExists($table)) { + return []; + } + $fields = $this->intersectExistingColumns($table, $candidateFields); + if (empty($fields)) { + return []; + } + $searchable = array_values(array_intersect($searchFields, $fields)); + if (empty($searchable)) { + return []; + } + + $wheres = []; + foreach ($fixedConditions as $column => $value) { + if (!in_array($column, $fields, true)) { + continue; + } + if ($value === null) { + $wheres[] = "`$column` IS NULL"; + } else { + $wheres[] = "`$column` = '" . $db->escape_string((string)$value) . "'"; + } + } + + if (!empty($customerNumbers) && $customerField !== null && in_array($customerField, $fields, true)) { + $wheres[] = "`$customerField` IN (" . implode(',', array_map('intval', $customerNumbers)) . ")"; + } + + $termClauses = []; + foreach ($terms as $term) { + $escaped = $db->escape_string($term); + foreach ($searchable as $field) { + $termClauses[] = "`$field` LIKE '%$escaped%'"; + } + } + if (!empty($termClauses)) { + $wheres[] = '(' . implode(' OR ', $termClauses) . ')'; + } + + if (empty($wheres)) { + return []; + } + + $sql = "SELECT " . implode(', ', array_map(fn($f) => "`$f`", $fields)) + . " FROM `$table`" + . " WHERE " . implode(' AND ', $wheres) + . " LIMIT " . $this->defaultEntityFetchLimit; + $result = $db->query($sql); + if (!($result instanceof \mysqli_result)) { + return []; + } + return $db->fetch_all($result); + } + + /** + * Generic join search helper. + * + * @param array $selectFields + * @param array $searchFields + * @param array $terms + * @return array> + */ + private function searchTableWithJoin( + string $table, + string $fromClause, + array $selectFields, + array $searchFields, + array $terms, + string $baseWhere + ): array { + global $db; + if (!$this->tableExists($table)) { + return []; + } + if (empty($terms)) { + return []; + } + $termClauses = []; + foreach ($terms as $term) { + $escaped = $db->escape_string($term); + foreach ($searchFields as $field) { + $termClauses[] = "$field LIKE '%$escaped%'"; + } + } + if (empty($termClauses)) { + return []; + } + $sql = "SELECT " . implode(', ', $selectFields) + . " FROM " . $fromClause + . " WHERE " . $baseWhere + . " AND (" . implode(' OR ', $termClauses) . ")" + . " LIMIT " . $this->defaultEntityFetchLimit; + $result = $db->query($sql); + if (!($result instanceof \mysqli_result)) { + return []; + } + return $db->fetch_all($result); + } + + private function scoreRow(array $row, array $fields, array $terms): int + { + $score = 0; + foreach ($terms as $term) { + $termLower = mb_strtolower($term); + foreach ($fields as $field) { + if (!array_key_exists($field, $row) || $row[$field] === null) { + continue; + } + $value = trim((string)$row[$field]); + if ($value === '') { + continue; + } + $valueLower = mb_strtolower($value); + if ($valueLower === $termLower) { + $score += 100; + continue; + } + if (str_starts_with($valueLower, $termLower)) { + $score += 60; + continue; + } + if (str_contains($valueLower, $termLower)) { + $score += 30; + continue; + } + if (strlen($termLower) >= 4 && strlen($valueLower) <= 64) { + $distance = levenshtein($termLower, $valueLower); + if ($distance <= 2) { + $score += 20 - ($distance * 5); + } + } + } + } + return $score; + } + + private function tableExists(string $table): bool + { + return !empty($this->getColumns($table)); + } + + private function intersectExistingColumns(string $table, array $candidateFields): array + { + $columns = $this->getColumns($table); + if (empty($columns)) { + return []; + } + return array_values(array_intersect($candidateFields, $columns)); + } + + private function getColumns(string $table): array + { + if (isset($this->tableColumnsCache[$table])) { + return $this->tableColumnsCache[$table]; + } + global $db; + try { + $result = $db->query("SHOW COLUMNS FROM `$table`"); + if (!($result instanceof \mysqli_result)) { + $this->tableColumnsCache[$table] = []; + return []; + } + $rows = $db->fetch_all($result); + $columns = array_values(array_map(static fn($row) => (string)$row['Field'], $rows)); + $this->tableColumnsCache[$table] = $columns; + return $columns; + } catch (Throwable) { + $this->tableColumnsCache[$table] = []; + return []; + } + } + + private function looksSecretVariable(string $variable): bool + { + $variable = mb_strtolower($variable); + return str_contains($variable, 'api_key') + || str_contains($variable, 'secret') + || str_contains($variable, 'password') + || str_contains($variable, 'token') + || str_contains($variable, 'private_key'); + } + + private function mergeResults(array $base, array $incoming): array + { + $indexed = []; + foreach ($base as $item) { + $key = (string)$item['entity_type'] . ':' . (string)$item['entity_id']; + $indexed[$key] = $item; + } + foreach ($incoming as $item) { + $key = (string)$item['entity_type'] . ':' . (string)$item['entity_id']; + if (!isset($indexed[$key])) { + $indexed[$key] = $item; + continue; + } + if ((int)$item['score'] > (int)$indexed[$key]['score']) { + $indexed[$key]['score'] = (int)$item['score']; + } + if (!isset($indexed[$key]['association_reason']) && isset($item['association_reason'])) { + $indexed[$key]['association_reason'] = $item['association_reason']; + } + } + return array_values($indexed); + } + + private function tokenize(string $query): array + { + $query = trim(mb_strtolower($query)); + if ($query === '') { + return []; + } + $parts = preg_split('/[^a-z0-9_]+/iu', $query) ?: []; + $parts = array_values(array_filter(array_map('trim', $parts), static fn($p) => $p !== '' && mb_strlen($p) >= 2)); + return array_values(array_unique($parts)); + } + + private function normalizeTypes(array $types): array + { + $normalized = []; + foreach ($types as $type) { + if (!is_string($type)) { + continue; + } + $t = trim(mb_strtolower($type)); + if ($t === '') { + continue; + } + $normalized[] = $t; + } + return array_values(array_unique($normalized)); + } + + private function allEntityTypes(): array + { + return [ + 'objects', + 'module_config', + 'orders', + 'order_items', + 'customers', + 'employees', + 'subusers', + 'customer_discounts', + 'customer_fixed_prices', + 'departments', + 'permissions', + 'roles', + 'invoices', + 'vehicles', + ]; + } + + private function taxonomy(array $activeTypes): array + { + $aliases = [ + 'customers' => ['customer', 'account', 'company'], + 'orders' => ['order', 'work order'], + 'order_items' => ['order item', 'line item'], + 'invoices' => ['invoice', 'billing'], + 'vehicles' => ['vehicle', 'truck', 'plate'], + 'employees' => ['employee', 'staff'], + 'subusers' => ['subuser', 'driver'], + 'customer_discounts' => ['discount', 'price override'], + 'customer_fixed_prices' => ['fixed price', 'monthly agreement'], + 'departments' => ['department', 'location'], + 'permissions' => ['permission', 'acl'], + 'roles' => ['role', 'group'], + 'module_config' => ['module config', 'setting', 'configuration'], + 'objects' => ['attachment', 'object'], + ]; + $taxonomy = []; + foreach ($activeTypes as $type) { + $taxonomy[$type] = $aliases[$type] ?? []; + } + return $taxonomy; + } + + private function groupResultsByType(array $results): array + { + $grouped = []; + foreach ($results as $item) { + $type = (string)$item['entity_type']; + if (!isset($grouped[$type])) { + $grouped[$type] = []; + } + $grouped[$type][] = $item; + } + return $grouped; + } +} diff --git a/services/nginx/app/cron/Cron.php b/services/nginx/app/cron/Cron.php index c3d12991..867f7fe6 100644 --- a/services/nginx/app/cron/Cron.php +++ b/services/nginx/app/cron/Cron.php @@ -3,6 +3,7 @@ use classes\backup_store; use classes\economic; +use classes\system_search_cache; use classes\xlvask; use classes\slack as Slack; use classes\email as Email; @@ -80,6 +81,12 @@ $cron_tasks = [ 'next_run' => 0, 'function' => 'SyncXLVaskModuleCron', ], + 'SystemSearchCacheMaintenanceCron' => [ + 'interval' => 300, // 5 minutes + 'last_run' => 0, + 'next_run' => 0, + 'function' => 'SystemSearchCacheMaintenanceCron', + ], 'GoalsProgressAlertsCron' => [ 'interval' => 60, // check every minute 'last_run' => 0, @@ -162,6 +169,28 @@ function SyncXLVaskModuleCron(): void } } +function SystemSearchCacheMaintenanceCron(): void +{ + try { + $rebuildRequest = system_search_cache::consumeRebuildRequest(); + $dirtyTables = system_search_cache::consumeDirtyTables(); + + if ($rebuildRequest !== null) { + system_search_cache::clearQueryCaches(); + system_search_cache::clearIntentCaches(); + echo "[" . date('Y-m-d H:i:s') . "][CRON] System search cache rebuild handled. Scope: " . ($rebuildRequest['scope'] ?? 'all') . "\n"; + return; + } + + if (!empty($dirtyTables)) { + system_search_cache::clearQueryCaches(); + echo "[" . date('Y-m-d H:i:s') . "][CRON] System search query cache invalidated for dirty tables: " . implode(', ', $dirtyTables) . "\n"; + } + } catch (Throwable $e) { + warn('SystemSearchCacheMaintenanceCron failed: ' . $e->getMessage()); + } +} + /** * GoalsProgressAlertsCron * @@ -417,4 +446,4 @@ foreach ( $cron_tasks as $task => $data ) { } else { $response_cron[] = $task . ' is not due to run yet, next run is at ' . date('Y-m-d H:i:s', $nextRun) . ' (' . ($nextRun - time()) . ' seconds)'; } -} \ No newline at end of file +} diff --git a/services/nginx/app/interfaces/system_search_intent_parser_i.php b/services/nginx/app/interfaces/system_search_intent_parser_i.php new file mode 100644 index 00000000..3bfcda25 --- /dev/null +++ b/services/nginx/app/interfaces/system_search_intent_parser_i.php @@ -0,0 +1,26 @@ +, + * entity_hints: array, + * confidence: float, + * association_hint: bool, + * fallback_reason: string|null, + * source: string + * } + */ + public function parse(string $query, array $allowedEntityTypes, array $taxonomy = []): array; +} + diff --git a/services/nginx/app/routes/systemSearchRoute.php b/services/nginx/app/routes/systemSearchRoute.php new file mode 100644 index 00000000..681d5f28 --- /dev/null +++ b/services/nginx/app/routes/systemSearchRoute.php @@ -0,0 +1,416 @@ +get('/search/system', function () { + $this->handleSearchRequest(); + }); + + $this->post('/search/system', function () { + $this->handleSearchRequest(); + }); + + $this->delete('/superuser/search/system/cache', function () { + global /** @var response $response */ + $response; + $this->requirePermission('superuser_search_system_cache_clear'); + + system_search_cache::clearAll(); + $response->success([ + 'message' => 'System search cache cleared', + 'query_cache_cleared' => true, + 'intent_cache_cleared' => true, + ]); + }, [ + 'superuser_search_system_cache_clear' => 'Clear system-wide search query and intent caches', + ]); + + $this->post('/superuser/search/system/cache/rebuild', function () { + global /** @var response $response */ + $response; + $this->requirePermission('superuser_search_system_cache_rebuild'); + + $params = $this->getRequestPayload(); + $scope = strtolower(trim((string)($params['scope'] ?? 'all'))); + $types = $this->parseTypeList($params['types'] ?? []); + $request = system_search_cache::enqueueRebuild($scope, $types); + + // Rebuild endpoint also clears parser namespace immediately. + system_search_cache::clearQueryCaches(); + system_search_cache::clearIntentCaches(); + + $response->success([ + 'message' => 'System search cache rebuild queued', + 'request' => $request, + 'query_cache_cleared' => true, + 'intent_cache_cleared' => true, + ]); + }, [ + 'superuser_search_system_cache_rebuild' => 'Queue and trigger a system-wide search cache rebuild', + ]); + } + + private function handleSearchRequest(): void + { + global /** @var response $response */ + /** @var router $router */ + $response, $router; + + $auth = new authentication(); + $user = $auth->get_user(); + $subuser = $auth->get_subuser(); + if ($user === false && $subuser === false) { + $response->error('Invalid session', 401); + } + + $params = $this->getRequestPayload(); + $query = trim((string)($params['q'] ?? $params['query'] ?? $params['search'] ?? '')); + if ($query === '') { + $response->error('Missing required parameter: query', 400); + } + + $includeTypes = $this->parseTypeList($params['include_types'] ?? []); + $excludeTypes = $this->parseTypeList($params['exclude_types'] ?? []); + $includeAssociations = $this->toBool($params['include_associations'] ?? true, true); + $debugIntent = $this->toBool($params['debug_intent'] ?? false, false); + $limit = $this->clampInt((int)($params['limit'] ?? 50), 1, 200, 50); + $offset = max(0, (int)($params['offset'] ?? 0)); + + [$allowedTypes, $ownOnlyTypes] = $this->resolveAllowedTypes(); + if (empty($allowedTypes)) { + $response->error('Permission denied. No searchable entity types available for this user.', 403); + } + + $permissionsCatalogAll = $this->flattenPermissionCatalog((array)$router->getPermissions()); + $permissionsCatalogOwn = []; + if ($user !== false) { + try { + $permissionsCatalogOwn = array_values(array_unique(array_map('strval', (array)$user->getGroup()->getPermissions()))); + } catch (Throwable) { + $permissionsCatalogOwn = []; + } + } + + $service = new system_search_service(); + $result = $service->search([ + 'query' => $query, + 'include_types' => $includeTypes, + 'exclude_types' => $excludeTypes, + 'allowed_types' => $allowedTypes, + 'own_only_types' => $ownOnlyTypes, + 'own_customer_number' => $this->resolveEffectiveCustomerNumber(), + 'permissions_catalog_all' => $permissionsCatalogAll, + 'permissions_catalog_own' => $permissionsCatalogOwn, + 'module_config_visibility' => $this->buildModuleConfigVisibility(), + 'include_associations' => $includeAssociations, + 'debug_intent' => $debugIntent, + 'limit' => $limit, + 'offset' => $offset, + ]); + + $response->success($result); + } + + private function getRequestPayload(): array + { + $payload = $this->getParametersAsArray(); + return is_array($payload) ? $payload : []; + } + + private function resolveAllowedTypes(): array + { + $allowed = []; + $ownOnly = []; + foreach ($this->entityPermissionMap() as $type => $permissionSets) { + $hasAll = $this->hasAnyPermission((array)($permissionSets['all'] ?? [])); + $hasOwn = $this->hasAnyPermission((array)($permissionSets['own'] ?? [])); + if (!$hasAll && !$hasOwn) { + continue; + } + $allowed[] = $type; + if (!$hasAll && $hasOwn) { + $ownOnly[] = $type; + } + } + return [ + array_values(array_intersect($this->allEntityTypes(), array_values(array_unique($allowed)))), + array_values(array_intersect($this->allEntityTypes(), array_values(array_unique($ownOnly)))), + ]; + } + + private function entityPermissionMap(): array + { + return [ + 'objects' => [ + 'all' => ['list_order_attachments', 'download_order_attachments', 'list_department_selfserve_task_attachments'], + 'own' => ['list_own_order_attachments', 'download_order_attachments_own'], + ], + 'module_config' => [ + 'all' => $this->moduleConfigPermissions(), + 'own' => [], + ], + 'orders' => [ + 'all' => ['list_orders', 'fetch_order'], + 'own' => ['list_own_orders', 'fetch_own_order'], + ], + 'order_items' => [ + 'all' => ['list_order_items'], + 'own' => ['list_own_order_items'], + ], + 'customers' => [ + 'all' => ['search_customers', 'list_users', 'get_user_from_customer_number'], + 'own' => ['user'], + ], + 'employees' => [ + 'all' => ['list_users'], + 'own' => [], + ], + 'subusers' => [ + 'all' => ['list_subuser_grants', 'manage_subuser_grants'], + 'own' => ['list_own_subusers', 'list_own_subuser_grants'], + ], + 'customer_discounts' => [ + 'all' => ['get_custom_prices_other', 'set_custom_price'], + 'own' => [], + ], + 'customer_fixed_prices' => [ + 'all' => ['get_customer_fixed_pricing'], + 'own' => [], + ], + 'departments' => [ + 'all' => ['list_departments'], + 'own' => [], + ], + 'permissions' => [ + 'all' => ['permissions_list'], + 'own' => ['permissions_list_own'], + ], + 'roles' => [ + 'all' => ['list_roles'], + 'own' => [], + ], + 'invoices' => [ + 'all' => ['list_collected_invoices', 'list_collected_invoices_economic_overview'], + 'own' => ['user_invoices'], + ], + 'vehicles' => [ + 'all' => ['list_vehicles_other', 'list_unknown_customer_vehicles'], + 'own' => ['list_own_vehicles'], + ], + ]; + } + + private function moduleConfigPermissions(): array + { + return [ + 'economic_config', + 'recaptcha_config', + 'email_config', + 'backups_config', + 'modules_bird_config', + 'motorapi_config', + 'stripe_config', + 'fxratesapi_config', + 'weatherapi_config', + 'gatewayapi_config', + 'xlvask_config', + 'entra_config', + 'modules_limble_config', + 'modules_ocrspace_config', + 'modules_openai_config', + 'modules_licenseplaterecognizer_config', + 'modules_virkdata_config', + 'modules_shelly_config', + 'modules_selfserve_config', + ]; + } + + private function buildModuleConfigVisibility(): array + { + global $db; + + $modulePermissions = [ + 'economic' => ['economic_config'], + 'reCAPTCHA' => ['recaptcha_config'], + 'Email' => ['email_config'], + 'Backups' => ['backups_config'], + 'bird' => ['modules_bird_config'], + 'motorapi' => ['motorapi_config'], + 'Stripe' => ['stripe_config'], + 'fxratesapi' => ['fxratesapi_config'], + 'weatherapi' => ['weatherapi_config'], + 'GatewayAPI' => ['gatewayapi_config'], + 'xlvask' => ['xlvask_config'], + 'Entra' => ['entra_config'], + 'limble' => ['modules_limble_config'], + 'ocrSpace' => ['modules_ocrspace_config'], + 'openAI' => ['modules_openai_config'], + 'licenseplaterecognizer' => ['modules_licenseplaterecognizer_config'], + 'virkdata' => ['modules_virkdata_config'], + 'shelly' => ['modules_shelly_config'], + 'selfserve' => ['modules_selfserve_config'], + ]; + + $visibility = []; + try { + $result = $db->query('SELECT DISTINCT module FROM module_config'); + if ($result instanceof \mysqli_result) { + $rows = $db->fetch_all($result); + foreach ($rows as $row) { + $module = (string)($row['module'] ?? ''); + if ($module === '') { + continue; + } + $candidates = $modulePermissions[$module] ?? []; + if (empty($candidates)) { + $slug = strtolower(preg_replace('/[^a-z0-9]+/i', '', $module) ?? ''); + if ($slug !== '') { + $candidates[] = $slug . '_config'; + $candidates[] = 'modules_' . $slug . '_config'; + } + } + $visibility[$module] = $this->hasAnyPermission($candidates); + } + } + } catch (Throwable) { + // Fail open for compatibility if module map cannot be loaded. + } + + return $visibility; + } + + private function hasAnyPermission(array $permissions): bool + { + foreach ($permissions as $permission) { + if (!is_string($permission) || $permission === '') { + continue; + } + if ($this->hasPermission($permission)) { + return true; + } + } + return false; + } + + private function flattenPermissionCatalog(array $permissions): array + { + $flat = []; + $walker = function (mixed $node) use (&$flat, &$walker): void { + if (!is_array($node)) { + return; + } + foreach ($node as $key => $value) { + if (is_string($key) && is_string($value)) { + $flat[$key] = $value; + continue; + } + if (is_array($value)) { + $walker($value); + } + } + }; + $walker($permissions); + return $flat; + } + + private function parseTypeList(mixed $value): array + { + $result = []; + $raw = []; + if (is_array($value)) { + $raw = $value; + } elseif (is_string($value)) { + $trimmed = trim($value); + if ($trimmed === '') { + return []; + } + if (str_starts_with($trimmed, '[')) { + $decoded = json_decode($trimmed, true); + if (is_array($decoded)) { + $raw = $decoded; + } else { + $raw = explode(',', $trimmed); + } + } else { + $raw = explode(',', $trimmed); + } + } elseif ($value !== null) { + $raw = [$value]; + } + + foreach ($raw as $item) { + if (!is_string($item)) { + continue; + } + $normalized = strtolower(trim($item)); + if ($normalized === '') { + continue; + } + $result[] = $normalized; + } + return array_values(array_unique($result)); + } + + private function toBool(mixed $value, bool $default): bool + { + if (is_bool($value)) { + return $value; + } + if (is_int($value) || is_float($value)) { + return (bool)$value; + } + if (is_string($value)) { + $parsed = filter_var($value, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); + return $parsed ?? $default; + } + return $default; + } + + private function clampInt(int $value, int $min, int $max, int $default): int + { + if ($value === 0) { + $value = $default; + } + if ($value < $min) { + return $min; + } + if ($value > $max) { + return $max; + } + return $value; + } + + private function allEntityTypes(): array + { + return [ + 'objects', + 'module_config', + 'orders', + 'order_items', + 'customers', + 'employees', + 'subusers', + 'customer_discounts', + 'customer_fixed_prices', + 'departments', + 'permissions', + 'roles', + 'invoices', + 'vehicles', + ]; + } +} diff --git a/services/nginx/app/tests/Integration/Search/SystemSearchCacheIntegrationTest.php b/services/nginx/app/tests/Integration/Search/SystemSearchCacheIntegrationTest.php new file mode 100644 index 00000000..36ca8737 --- /dev/null +++ b/services/nginx/app/tests/Integration/Search/SystemSearchCacheIntegrationTest.php @@ -0,0 +1,157 @@ +store[$key] ?? null; + } + + public function set(string $key, string $value): void + { + $this->store[$key] = $value; + } + + public function setEx(string $key, string $value, int $ttl): void + { + $this->store[$key] = $value; + } + + public function delete(string $key): void + { + unset($this->store[$key]); + } + + public function expire(string $key, int $ttl): void + { + // TTL is not simulated in this test adapter. + } + + public function set_array(string $key, array $value): void + { + $this->store[$key] = json_encode($value, JSON_UNESCAPED_UNICODE); + } + + public function get_array(string $key): ?array + { + $value = $this->store[$key] ?? null; + if (!is_string($value)) { + return null; + } + $decoded = json_decode($value, true); + return is_array($decoded) ? $decoded : null; + } + + public function clear_keys(string $pattern): void + { + $regex = '/^' . str_replace('\*', '.*', preg_quote($pattern, '/')) . '$/'; + foreach (array_keys($this->store) as $key) { + if (preg_match($regex, $key)) { + unset($this->store[$key]); + } + } + } + } +} + +beforeEach(function (): void { + system_search_cache::setAdapterForTests(new SystemSearchTestRedisAdapter()); +}); + +afterEach(function (): void { + system_search_cache::setAdapterForTests(null); +}); + +it('reuses parser cache entries for repeated natural-language intent requests', function (): void { + $calls = 0; + $parser = new system_search_openai_intent_parser( + function (array $payload, string $apiKey) use (&$calls): array { + $calls++; + return [ + 'output' => [ + [ + 'content' => [ + [ + 'text' => json_encode([ + 'success' => true, + 'normalized_query' => 'acme unpaid invoices', + 'aliases' => ['acme'], + 'entity_hints' => ['invoices'], + 'confidence' => 0.91, + 'association_hint' => true, + ], JSON_UNESCAPED_UNICODE), + ], + ], + ], + ], + ]; + }, + true, + 'test-key' + ); + + $first = $parser->parse('find unpaid invoices for acme', ['invoices', 'customers']); + $second = $parser->parse('find unpaid invoices for acme', ['invoices', 'customers']); + + expect($first['source'])->toBe('openai'); + expect($second['source'])->toBe('cache'); + expect($calls)->toBe(1); +}); + +it('clears parser cache namespace via clearAll to force a fresh parse', function (): void { + $calls = 0; + $parser = new system_search_openai_intent_parser( + function (array $payload, string $apiKey) use (&$calls): array { + $calls++; + return [ + 'output' => [ + [ + 'content' => [ + [ + 'text' => json_encode([ + 'success' => true, + 'normalized_query' => 'acme invoices', + 'aliases' => ['acme'], + 'entity_hints' => ['invoices'], + 'confidence' => 0.8, + 'association_hint' => false, + ], JSON_UNESCAPED_UNICODE), + ], + ], + ], + ], + ]; + }, + true, + 'test-key' + ); + + $parser->parse('acme invoices', ['invoices']); + system_search_cache::clearAll(); + $result = $parser->parse('acme invoices', ['invoices']); + + expect($result['source'])->toBe('openai'); + expect($calls)->toBe(2); +}); + +it('invalidates query cache immediately when dirty-table marker is registered', function (): void { + $hash = md5('test-query'); + system_search_cache::setQuery($hash, ['results' => [], 'grouped_results' => [], 'meta' => []], 120); + expect(system_search_cache::getQuery($hash))->not->toBeNull(); + + system_search_cache::markDirtyTable('orders'); + $dirty = system_search_cache::consumeDirtyTables(); + + expect(system_search_cache::getQuery($hash))->toBeNull(); + expect($dirty)->toContain('orders'); +}); diff --git a/services/nginx/app/tests/Unit/Search/SystemSearchInvalidationHooksTest.php b/services/nginx/app/tests/Unit/Search/SystemSearchInvalidationHooksTest.php new file mode 100644 index 00000000..96bb68bd --- /dev/null +++ b/services/nginx/app/tests/Unit/Search/SystemSearchInvalidationHooksTest.php @@ -0,0 +1,32 @@ +not->toBeFalse(); + expect($content)->toContain('markSystemSearchDirtyTable'); + expect($content)->toContain('system_search_cache::markDirtyTable'); +}); + +it('marks system search cache dirty from object property mutations', function (): void { + $content = file_get_contents(app_path('classes/object_property.php')); + + expect($content)->not->toBeFalse(); + expect($content)->toContain("system_search_cache::markDirtyTable(\$this->table)"); +}); + +it('marks system search cache dirty when module config values change', function (): void { + $content = file_get_contents(app_path('traits/module_config_variable_t.php')); + + expect($content)->not->toBeFalse(); + expect($content)->toContain("system_search_cache::markDirtyTable('module_config')"); +}); + +it('registers cron maintenance task for system search cache', function (): void { + $content = file_get_contents(app_path('cron/Cron.php')); + + expect($content)->not->toBeFalse(); + expect($content)->toContain('SystemSearchCacheMaintenanceCron'); + expect($content)->toContain("system_search_cache::consumeRebuildRequest()"); + expect($content)->toContain("system_search_cache::consumeDirtyTables()"); +}); diff --git a/services/nginx/app/tests/Unit/Search/SystemSearchOpenAiIntentParserTest.php b/services/nginx/app/tests/Unit/Search/SystemSearchOpenAiIntentParserTest.php new file mode 100644 index 00000000..556dd7f4 --- /dev/null +++ b/services/nginx/app/tests/Unit/Search/SystemSearchOpenAiIntentParserTest.php @@ -0,0 +1,192 @@ +store = []; + } + + public function get(string $key): mixed + { + return $this->store[$key] ?? null; + } + + public function set(string $key, string $value): void + { + $this->store[$key] = $value; + } + + public function setEx(string $key, string $value, int $ttl): void + { + $this->store[$key] = $value; + } + + public function delete(string $key): void + { + unset($this->store[$key]); + } + + public function expire(string $key, int $ttl): void + { + // TTL is not simulated in unit tests. + } + + public function set_array(string $key, array $value): void + { + $this->store[$key] = json_encode($value, JSON_UNESCAPED_UNICODE); + } + + public function get_array(string $key): ?array + { + $value = $this->store[$key] ?? null; + if (!is_string($value)) { + return null; + } + $decoded = json_decode($value, true); + return is_array($decoded) ? $decoded : null; + } + + public function clear_keys(string $pattern): void + { + $regex = '/^' . str_replace('\*', '.*', preg_quote($pattern, '/')) . '$/'; + foreach (array_keys($this->store) as $key) { + if (preg_match($regex, $key)) { + unset($this->store[$key]); + } + } + } + } +} + +beforeEach(function (): void { + system_search_cache::setAdapterForTests(new SystemSearchTestRedisAdapter()); +}); + +afterEach(function (): void { + system_search_cache::setAdapterForTests(null); +}); + +it('redacts obvious sensitive fragments before sending query to intent parser', function (): void { + $query = 'Contact alice@example.com at +45 12 34 56 78, cvr 12345678'; + $redacted = system_search_openai_intent_parser::redactSensitiveQuery($query); + + expect($redacted)->toContain('[email]'); + expect($redacted)->toContain('[phone]'); + expect($redacted)->toContain('[cvr]'); + expect($redacted)->not->toContain('alice@example.com'); + expect($redacted)->not->toContain('12345678'); +}); + +it('builds payload with redacted query and parses strict JSON output', function (): void { + $capturedPrompt = ''; + $parser = new system_search_openai_intent_parser( + function (array $payload, string $apiKey) use (&$capturedPrompt): array { + $capturedPrompt = (string)($payload['input'][0]['content'][0]['text'] ?? ''); + return [ + 'output' => [ + [ + 'content' => [ + [ + 'text' => json_encode([ + 'success' => true, + 'normalized_query' => 'acme unpaid invoices', + 'aliases' => ['acme', 'invoice overdue'], + 'entity_hints' => ['customers', 'invoices'], + 'confidence' => 0.93, + 'association_hint' => true, + ], JSON_UNESCAPED_UNICODE), + ], + ], + ], + ], + ]; + }, + true, + 'test-key' + ); + + $result = $parser->parse( + 'find unpaid invoices for alice@example.com', + ['customers', 'invoices'], + ['customers' => ['account'], 'invoices' => ['billing']] + ); + + expect($capturedPrompt)->toContain('[email]'); + expect($capturedPrompt)->not->toContain('alice@example.com'); + expect($result['success'])->toBeTrue(); + expect($result['source'])->toBe('openai'); + expect($result['confidence'])->toBe(0.93); + expect($result['entity_hints'])->toBe(['customers', 'invoices']); +}); + +it('falls back safely when OpenAI is disabled', function (): void { + $parser = new system_search_openai_intent_parser(null, false, null); + $result = $parser->parse('find acme invoices', ['customers', 'invoices']); + + expect($result['success'])->toBeFalse(); + expect($result['source'])->toBe('none'); + expect($result['fallback_reason'])->toBe('openai_disabled'); +}); + +it('handles malformed OpenAI response payloads without throwing', function (): void { + $parser = new system_search_openai_intent_parser( + function (array $payload, string $apiKey): array { + return ['output' => []]; + }, + true, + 'test-key' + ); + + $result = $parser->parse('find acme invoices', ['invoices']); + + expect($result['success'])->toBeFalse(); + expect($result['source'])->toBe('openai'); + expect((string)$result['fallback_reason'])->toContain('Invalid response format'); +}); + +it('uses parser cache for identical query and allowed type combinations', function (): void { + $calls = 0; + $parser = new system_search_openai_intent_parser( + function (array $payload, string $apiKey) use (&$calls): array { + $calls++; + return [ + 'output' => [ + [ + 'content' => [ + [ + 'text' => json_encode([ + 'success' => true, + 'normalized_query' => 'acme invoices', + 'aliases' => ['acme'], + 'entity_hints' => ['invoices'], + 'confidence' => 0.8, + 'association_hint' => false, + ], JSON_UNESCAPED_UNICODE), + ], + ], + ], + ], + ]; + }, + true, + 'test-key' + ); + + $first = $parser->parse('acme invoices', ['invoices']); + $second = $parser->parse('acme invoices', ['invoices']); + + expect($first['source'])->toBe('openai'); + expect($second['source'])->toBe('cache'); + expect($calls)->toBe(1); +}); diff --git a/services/nginx/app/tests/Unit/Search/SystemSearchOpenApiSpecTest.php b/services/nginx/app/tests/Unit/Search/SystemSearchOpenApiSpecTest.php new file mode 100644 index 00000000..1f7ebe88 --- /dev/null +++ b/services/nginx/app/tests/Unit/Search/SystemSearchOpenApiSpecTest.php @@ -0,0 +1,38 @@ +markTestSkipped('openapi.yaml is not available in this runtime environment.'); +} + +it('documents system-wide search endpoints in openapi', function (): void { + $content = system_search_openapi_content_or_skip(); + + expect($content)->toContain('/search/system:'); + expect($content)->toContain('/superuser/search/system/cache:'); + expect($content)->toContain('/superuser/search/system/cache/rebuild:'); +}); + +it('documents debug_intent and parser metadata schema in openapi', function (): void { + $content = system_search_openapi_content_or_skip(); + + expect($content)->toContain('debug_intent:'); + expect($content)->toContain('SystemSearchIntentParserMeta:'); + expect($content)->toContain('intent_parser:'); + expect($content)->toContain('SystemSearchResponse:'); +}); diff --git a/services/nginx/app/tests/Unit/Search/SystemSearchRouteWiringTest.php b/services/nginx/app/tests/Unit/Search/SystemSearchRouteWiringTest.php new file mode 100644 index 00000000..c14e1b5d --- /dev/null +++ b/services/nginx/app/tests/Unit/Search/SystemSearchRouteWiringTest.php @@ -0,0 +1,37 @@ +not->toBeFalse(); + expect($content)->toContain('/search/system'); + expect($content)->toContain("'debug_intent'"); + expect($content)->toContain("'include_types'"); + expect($content)->toContain("'exclude_types'"); + expect($content)->toContain('new system_search_service()'); +}); + +it('registers superuser cache clear and rebuild endpoints for system search', function (): void { + $routeFile = app_path('routes/systemSearchRoute.php'); + $content = file_get_contents($routeFile); + + expect($content)->not->toBeFalse(); + expect($content)->toContain('/superuser/search/system/cache'); + expect($content)->toContain('/superuser/search/system/cache/rebuild'); + expect($content)->toContain("requirePermission('superuser_search_system_cache_clear')"); + expect($content)->toContain("requirePermission('superuser_search_system_cache_rebuild')"); + expect($content)->toContain('system_search_cache::clearIntentCaches()'); +}); + +it('passes permission and own-scope context into system search service', function (): void { + $routeFile = app_path('routes/systemSearchRoute.php'); + $content = file_get_contents($routeFile); + + expect($content)->not->toBeFalse(); + expect($content)->toContain("'allowed_types'"); + expect($content)->toContain("'own_only_types'"); + expect($content)->toContain("'own_customer_number'"); + expect($content)->toContain("'permissions_catalog_all'"); + expect($content)->toContain("'permissions_catalog_own'"); +}); diff --git a/services/nginx/app/tests/Unit/Search/SystemSearchServiceIntentFlowTest.php b/services/nginx/app/tests/Unit/Search/SystemSearchServiceIntentFlowTest.php new file mode 100644 index 00000000..774c5d53 --- /dev/null +++ b/services/nginx/app/tests/Unit/Search/SystemSearchServiceIntentFlowTest.php @@ -0,0 +1,224 @@ +> + */ + public array $responses = []; + + public function __construct(array $responses = []) + { + $this->responses = $responses; + } + + public function parse(string $query, array $allowedEntityTypes, array $taxonomy = []): array + { + $this->calls++; + if (empty($this->responses)) { + return [ + 'success' => false, + 'normalized_query' => '', + 'aliases' => [], + 'entity_hints' => [], + 'confidence' => 0.0, + 'association_hint' => false, + 'fallback_reason' => 'no_response', + 'source' => 'none', + ]; + } + return array_shift($this->responses); + } + } +} + +if (!class_exists('TestableSystemSearchService')) { + class TestableSystemSearchService extends system_search_service + { + /** + * @var array> + */ + public array $lexicalCalls = []; + /** + * @var array>> + */ + private array $queuedLexicalResults; + + public function __construct(system_search_intent_parser_i $intentParser, array $queuedLexicalResults) + { + $this->queuedLexicalResults = $queuedLexicalResults; + parent::__construct($intentParser); + } + + protected function executeLexicalSearch( + array $activeTypes, + array $terms, + array $entityBoost, + array $ownOnlyTypes, + ?int $ownCustomerNumber, + array $permissionsCatalogAll, + array $permissionsCatalogOwn, + array $moduleConfigVisibility, + array $forcedCustomerNumbers = [] + ): array { + $this->lexicalCalls[] = [ + 'activeTypes' => $activeTypes, + 'terms' => $terms, + 'entityBoost' => $entityBoost, + 'ownOnlyTypes' => $ownOnlyTypes, + 'ownCustomerNumber' => $ownCustomerNumber, + 'forcedCustomerNumbers' => $forcedCustomerNumbers, + ]; + if (empty($this->queuedLexicalResults)) { + return []; + } + return array_shift($this->queuedLexicalResults); + } + } +} + +beforeEach(function (): void { + system_search_cache::setAdapterForTests(null); +}); + +it('does not invoke intent parser when lexical confidence is already high', function (): void { + $parser = new FakeSystemSearchIntentParser([ + [ + 'success' => true, + 'normalized_query' => 'ignored', + 'aliases' => ['ignored'], + 'entity_hints' => ['orders'], + 'confidence' => 0.99, + 'association_hint' => false, + 'fallback_reason' => null, + 'source' => 'openai', + ], + ]); + + $service = new TestableSystemSearchService($parser, [[ + ['entity_type' => 'orders', 'entity_id' => '1', 'title' => 'Order #1', 'score' => 95], + ['entity_type' => 'orders', 'entity_id' => '2', 'title' => 'Order #2', 'score' => 90], + ['entity_type' => 'orders', 'entity_id' => '3', 'title' => 'Order #3', 'score' => 88], + ['entity_type' => 'orders', 'entity_id' => '4', 'title' => 'Order #4', 'score' => 84], + ['entity_type' => 'orders', 'entity_id' => '5', 'title' => 'Order #5', 'score' => 80], + ]]); + + $result = $service->search([ + 'query' => 'order 1', + 'allowed_types' => ['orders'], + 'include_associations' => false, + 'debug_intent' => true, + ]); + + expect($parser->calls)->toBe(0); + expect($result['meta']['intent_parser']['status'])->toBe('skipped'); + expect(count($service->lexicalCalls))->toBe(1); +}); + +it('invokes parser on low-confidence lexical results and applies hints-only boosts', function (): void { + $parser = new FakeSystemSearchIntentParser([ + [ + 'success' => true, + 'normalized_query' => 'acme invoices', + 'aliases' => ['acme corp', 'invoice overdue'], + 'entity_hints' => ['orders'], + 'confidence' => 0.87, + 'association_hint' => true, + 'fallback_reason' => null, + 'source' => 'openai', + ], + ]); + + $service = new TestableSystemSearchService($parser, [ + [['entity_type' => 'orders', 'entity_id' => '1', 'title' => 'weak', 'score' => 20]], + [['entity_type' => 'orders', 'entity_id' => '2', 'title' => 'strong', 'score' => 85]], + ]); + + $result = $service->search([ + 'query' => 'acm inv', + 'allowed_types' => ['orders', 'customers'], + 'include_associations' => false, + 'debug_intent' => true, + ]); + + expect($parser->calls)->toBe(1); + expect(count($service->lexicalCalls))->toBe(2); + expect($service->lexicalCalls[1]['entityBoost']['orders'] ?? 0)->toBe(25); + expect(implode(' ', $service->lexicalCalls[1]['terms']))->toContain('acme'); + expect($result['meta']['intent_parser']['status'])->toBe('ok'); + expect($result['meta']['intent_parser']['source'])->toBe('openai'); +}); + +it('never lets parser entity hints override explicit include filters', function (): void { + $parser = new FakeSystemSearchIntentParser([ + [ + 'success' => true, + 'normalized_query' => 'customer acme', + 'aliases' => ['acme'], + 'entity_hints' => ['orders'], + 'confidence' => 0.7, + 'association_hint' => true, + 'fallback_reason' => null, + 'source' => 'openai', + ], + ]); + + $service = new TestableSystemSearchService($parser, [ + [['entity_type' => 'customers', 'entity_id' => '55', 'title' => 'Acme', 'score' => 10]], + [['entity_type' => 'customers', 'entity_id' => '55', 'title' => 'Acme', 'score' => 90]], + ]); + + $result = $service->search([ + 'query' => 'acm', + 'include_types' => ['customers'], + 'allowed_types' => ['customers', 'orders'], + 'include_associations' => false, + 'debug_intent' => true, + ]); + + expect($parser->calls)->toBe(1); + expect($service->lexicalCalls[0]['activeTypes'])->toBe(['customers']); + expect($service->lexicalCalls[1]['activeTypes'])->toBe(['customers']); + expect($result['results'][0]['entity_type'])->toBe('customers'); +}); + +it('returns fallback parser metadata when parser fails gracefully', function (): void { + $parser = new FakeSystemSearchIntentParser([ + [ + 'success' => false, + 'normalized_query' => '', + 'aliases' => [], + 'entity_hints' => [], + 'confidence' => 0.0, + 'association_hint' => false, + 'fallback_reason' => 'openai_disabled', + 'source' => 'none', + ], + ]); + + $service = new TestableSystemSearchService($parser, [ + [['entity_type' => 'orders', 'entity_id' => '1', 'title' => 'weak', 'score' => 5]], + ]); + + $result = $service->search([ + 'query' => 'unknown phrase', + 'allowed_types' => ['orders'], + 'include_associations' => false, + 'debug_intent' => true, + ]); + + expect($parser->calls)->toBe(1); + expect($result['meta']['intent_parser']['status'])->toBe('fallback'); + expect($result['meta']['intent_parser']['fallback_reason'])->toBe('openai_disabled'); +}); diff --git a/services/nginx/app/traits/db_object_t.php b/services/nginx/app/traits/db_object_t.php index 24b84a7a..110a73a0 100644 --- a/services/nginx/app/traits/db_object_t.php +++ b/services/nginx/app/traits/db_object_t.php @@ -8,6 +8,7 @@ use classes\attachments; use classes\db; use classes\object_property; use classes\response; +use classes\system_search_cache; use Exception; use mysqli_result; use objects\bookings_new_o; @@ -47,6 +48,7 @@ use objects\tokens_o; use objects\user_key_value_pairs_o; use objects\user_price_overrides_o; use objects\users_o; +use Throwable; trait db_object_t { @@ -67,6 +69,20 @@ trait db_object_t */ private string $additionalWhereClause = ''; // Additional where clause to add to the pagination query, this is used to add custom where clauses to the pagination query + private function markSystemSearchDirtyTable(?string $table = null): void + { + try { + $target = $table ?? $this->table; + $target = trim((string)$target, " `\t\n\r\0\x0B"); + if ($target === '') { + return; + } + system_search_cache::markDirtyTable($target); + } catch (Throwable) { + // Search invalidation must never block write operations. + } + } + public function __construct() { $this->structure(); @@ -289,7 +305,11 @@ trait db_object_t // Execute the update query $sql = "UPDATE $table SET $set WHERE $where"; - return $db->query($sql); + $result = $db->query($sql); + if ($result !== false) { + $this->markSystemSearchDirtyTable($table); + } + return $result; } /** @@ -1116,6 +1136,7 @@ trait db_object_t // Delete the object from the database self::delete_object($this->table, $this->id); } + $this->markSystemSearchDirtyTable(); // Trigger the object changed event self::objectChanged(); } @@ -1192,6 +1213,7 @@ trait db_object_t if (empty($set)) { return; } + $this->markSystemSearchDirtyTable(); // Trigger the object changed event self::objectChanged(); } @@ -1226,6 +1248,10 @@ trait db_object_t global $db; $sql = "DELETE FROM $table WHERE id = $id"; $db->query($sql); + try { + system_search_cache::markDirtyTable(trim((string)$table, " `\t\n\r\0\x0B")); + } catch (Throwable) { + } } /** @@ -1237,6 +1263,7 @@ trait db_object_t self::requireSelected(); // Delete the object from the database self::delete_object($this->table, $this->id); + $this->markSystemSearchDirtyTable(); // Trigger the object changed event self::objectChanged(); } @@ -1329,6 +1356,7 @@ trait db_object_t $set = implode(', ', $set); $sql = "INSERT INTO $this->table SET $set"; $db->query($sql); + $this->markSystemSearchDirtyTable(); return $db->insert_id(); } catch (Exception $e) { throw new Exception($e->getMessage()); @@ -1368,6 +1396,7 @@ trait db_object_t $id = $this->id; $sql = "UPDATE $this->table SET deleted_at = NULL WHERE id = $id"; $db->query($sql); + $this->markSystemSearchDirtyTable(); // Trigger the object changed event self::objectChanged(); } @@ -1426,4 +1455,4 @@ trait db_object_t $attachment = new attachments(); return $attachment->get($attachmentId); } -} \ No newline at end of file +} diff --git a/services/nginx/app/traits/module_config_variable_t.php b/services/nginx/app/traits/module_config_variable_t.php index cc456045..0df2f7f6 100644 --- a/services/nginx/app/traits/module_config_variable_t.php +++ b/services/nginx/app/traits/module_config_variable_t.php @@ -2,6 +2,7 @@ namespace traits; +use classes\system_search_cache; use Exception; trait module_config_variable @@ -87,6 +88,7 @@ trait module_config_variable global $db; $sql = "INSERT INTO module_config (module, variable, value, type) VALUES ('$module', '$variable', '$value', '" . self::getVariableType() . "')"; $db->query($sql); + system_search_cache::markDirtyTable('module_config'); } /** @@ -227,6 +229,7 @@ trait module_config_variable global $db; $sql = "UPDATE module_config SET value = '$value' WHERE module = '$module' AND variable = '$variable'"; $db->query($sql); + system_search_cache::markDirtyTable('module_config'); } /** @@ -300,4 +303,4 @@ trait module_config_variable { return $this->config_variable_required; } -} \ No newline at end of file +}