fix: remove embedded testmap data from test testsite payloads

Keep test definition responses focused on core fields and update test-map OpenAPI contracts/tests to match the new mapping flow.
This commit is contained in:
mahdahar 2026-04-06 11:24:58 +07:00
parent 694c5a6211
commit ae56e34885
9 changed files with 75 additions and 1209 deletions

View File

@ -17,38 +17,41 @@ class TestMapController extends BaseController {
public function __construct() {
$this->db = \Config\Database::connect();
$this->model = new TestMapModel;
$this->modelDetail = new TestMapDetailModel;
$this->model = new TestMapModel;
$this->modelDetail = new TestMapDetailModel;
$this->rules = [
'TestCode' => 'required',
'HostID' => 'required|integer',
'ClientID' => 'required|integer',
];
}
public function index() {
$rows = $this->model->getUniqueGroupings();
if (empty($rows)) { return $this->respond([ 'status' => 'success', 'message' => "no Data.", 'data' => [] ], 200); }
$rows = ValueSet::transformLabels($rows, [
'HostType' => 'entity_type',
'ClientType' => 'entity_type',
]);
return $this->respond([ 'status' => 'success', 'message'=> "Data fetched successfully", 'data' => $rows ], 200);
}
public function index() {
$rows = $this->model->getUniqueGroupings();
if (empty($rows)) { return $this->respond([ 'status' => 'success', 'message' => "no Data.", 'data' => [] ], 200); }
$rows = ValueSet::transformLabels($rows, [
'HostType' => 'entity_type',
'ClientType' => 'entity_type',
]);
$rows = array_map([$this, 'sanitizeTopLevelPayload'], $rows);
return $this->respond([ 'status' => 'success', 'message'=> "Data fetched successfully", 'data' => $rows ], 200);
}
public function show($id = null) {
$row = $this->model->where('TestMapID',$id)->where('EndDate', null)->first();
if (empty($row)) { return $this->respond([ 'status' => 'success', 'message' => "no Data.", 'data' => null ], 200); }
$row = ValueSet::transformLabels([$row], [
'HostType' => 'entity_type',
'ClientType' => 'entity_type',
])[0];
// Include testmapdetail records
$row['details'] = $this->modelDetail->getDetailsByTestMap($id);
$row = ValueSet::transformLabels([$row], [
'HostType' => 'entity_type',
'ClientType' => 'entity_type',
])[0];
$row = $this->sanitizeTopLevelPayload($row);
// Include testmapdetail records
$row['details'] = $this->modelDetail->getDetailsByTestMap($id);
return $this->respond([ 'status' => 'success', 'message'=> "Data fetched successfully", 'data' => $row ], 200);
}
@ -120,8 +123,16 @@ class TestMapController extends BaseController {
'ClientType' => 'entity_type',
]);
$rows = array_map([$this, 'sanitizeTopLevelPayload'], $rows);
return $this->respond([ 'status' => 'success', 'message'=> "Data fetched successfully", 'data' => $rows ], 200);
}
}
private function sanitizeTopLevelPayload(array $row): array
{
unset($row['TestCode'], $row['testcode']);
return $row;
}
public function batchCreate() {
$items = $this->request->getJSON(true);

View File

@ -111,12 +111,15 @@ class TestsController extends BaseController
$row['reftxt'] = $this->modelRefTxt->getFormattedByTestSiteID($id);
}
}
return $this->respond([
'status' => 'success',
'message' => 'Data fetched successfully',
'data' => $row,
], 200);
// Keep /api/test payload focused on test definition fields.
unset($row['testmap']);
return $this->respond([
'status' => 'success',
'message' => 'Data fetched successfully',
'data' => $row,
], 200);
}
public function create()
@ -390,12 +393,8 @@ class TestsController extends BaseController
break;
case 'TITLE':
if (isset($input['testmap']) && is_array($input['testmap'])) {
$this->saveTestMap($testSiteID, $testSiteCode, $input['testmap'], $action);
}
break;
case 'TITLE':
break;
case 'TEST':
case 'PARAM':
@ -418,10 +417,7 @@ class TestsController extends BaseController
break;
}
if ((TestValidationService::isTechnicalTest($typeCode) || TestValidationService::isCalc($typeCode)) && isset($input['testmap']) && is_array($input['testmap'])) {
$this->saveTestMap($testSiteID, $testSiteCode, $input['testmap'], $action);
}
}
}
private function saveTechDetails($testSiteID, $data, $action, $typeCode)
{

View File

@ -145,43 +145,32 @@ class TestDefSiteModel extends BaseModel {
$typeCode = $row['TestType'] ?? '';
if (TestValidationService::isCalc($typeCode)) {
$row['testdefcal'] = $db->table('testdefcal')
->select('testdefcal.*, d.DisciplineName, dept.DepartmentName')
->join('discipline d', 'd.DisciplineID=testdefcal.DisciplineID', 'left')
->join('department dept', 'dept.DepartmentID=testdefcal.DepartmentID', 'left')
->where('testdefcal.TestSiteID', $TestSiteID)
->where('testdefcal.EndDate IS NULL')
->get()->getResultArray();
$testMapModel = new \App\Models\Test\TestMapModel();
$row['testmap'] = $testMapModel->getMappingsByTestCode($row['TestSiteCode']);
} elseif (TestValidationService::isGroup($typeCode)) {
$testDefGrpModel = new \App\Models\Test\TestDefGrpModel();
$row['testdefgrp'] = $testDefGrpModel->getGroupMembers($TestSiteID);
$testMapModel = new \App\Models\Test\TestMapModel();
$row['testmap'] = $testMapModel->getMappingsByTestCode($row['TestSiteCode']);
} elseif (TestValidationService::isTitle($typeCode)) {
$testMapModel = new \App\Models\Test\TestMapModel();
$row['testmap'] = $testMapModel->getMappingsByTestCode($row['TestSiteCode']);
} elseif (TestValidationService::isTechnicalTest($typeCode)) {
// Technical details are now flattened into the main row
if ($row['DisciplineID']) {
$discipline = $db->table('discipline')->where('DisciplineID', $row['DisciplineID'])->get()->getRowArray();
$row['DisciplineName'] = $discipline['DisciplineName'] ?? null;
if (TestValidationService::isCalc($typeCode)) {
$row['testdefcal'] = $db->table('testdefcal')
->select('testdefcal.*, d.DisciplineName, dept.DepartmentName')
->join('discipline d', 'd.DisciplineID=testdefcal.DisciplineID', 'left')
->join('department dept', 'dept.DepartmentID=testdefcal.DepartmentID', 'left')
->where('testdefcal.TestSiteID', $TestSiteID)
->where('testdefcal.EndDate IS NULL')
->get()->getResultArray();
} elseif (TestValidationService::isGroup($typeCode)) {
$testDefGrpModel = new \App\Models\Test\TestDefGrpModel();
$row['testdefgrp'] = $testDefGrpModel->getGroupMembers($TestSiteID);
} elseif (TestValidationService::isTitle($typeCode)) {
} elseif (TestValidationService::isTechnicalTest($typeCode)) {
// Technical details are now flattened into the main row
if ($row['DisciplineID']) {
$discipline = $db->table('discipline')->where('DisciplineID', $row['DisciplineID'])->get()->getRowArray();
$row['DisciplineName'] = $discipline['DisciplineName'] ?? null;
}
if ($row['DepartmentID']) {
$department = $db->table('department')->where('DepartmentID', $row['DepartmentID'])->get()->getRowArray();
$row['DepartmentName'] = $department['DepartmentName'] ?? null;
}
$testMapModel = new \App\Models\Test\TestMapModel();
$row['testmap'] = $testMapModel->getMappingsByTestCode($row['TestSiteCode']);
}
if ($row['DepartmentID']) {
$department = $db->table('department')->where('DepartmentID', $row['DepartmentID'])->get()->getRowArray();
$row['DepartmentName'] = $department['DepartmentName'] ?? null;
}
}
return $row;
}

View File

@ -1,352 +0,0 @@
# Audit Logging Strategy (Implementation Ready)
## 1) Purpose, Scope, and Non-Goals
This document defines the production audit logging contract for CLQMS.
### Purpose
- Provide a single, normalized audit model for compliance, investigations, and operations.
- Ensure every protected workflow writes consistent, queryable audit records.
- Make behavior deterministic across API controllers, services, jobs, and integrations.
### Scope
This applies to four log tables:
- `logpatient` - patient identity, demographics, consent, insurance, and visit/ADT events.
- `logorder` - orders, specimen lifecycle, results lifecycle, and QC.
- `logmaster` - test/master configuration, value sets, role/permission updates, infrastructure configuration.
- `logsystem` - authentication, authorization, import/export, jobs, and system integrity operations.
### Non-goals
- This is not a replacement for metrics/tracing systems (Prometheus, APM, etc.).
- This is not a full immutable ledger; tamper evidence is implemented with controls described below.
## 2) Table Ownership
Use this mapping to choose the target table and minimum event shape.
| Event family | Table | Minimum keys in `Context` | Example `EventID` |
| --- | --- | --- | --- |
| Patient create/update/merge | `logpatient` | `route`, `request_id`, `entity_version` | `PATIENT_REGISTERED` |
| Consent/insurance changes | `logpatient` | `consent_type` or `payer_id` | `PATIENT_CONSENT_UPDATED` |
| Visit ADT transitions | `logpatient` | `visit_id`, `from_status`, `to_status` | `VISIT_TRANSFERRED` |
| Order create/cancel/reopen | `logorder` | `order_id`, `priority`, `source` | `ORDER_CREATED` |
| Specimen lifecycle | `logorder` | `specimen_id`, `specimen_status` | `SPECIMEN_RECEIVED` |
| Result lifecycle | `logorder` | `result_id`, `verification_state` | `RESULT_AMENDED` |
| QC lifecycle | `logorder` | `qc_run_id`, `instrument_id` | `QC_RECORDED` |
| Value sets/test definitions | `logmaster` | `config_group`, `change_ticket` | `VALUESET_ITEM_RETIRED` |
| Roles/permissions/users | `logmaster` | `target_user_id`, `target_role` | `USER_ROLE_CHANGED` |
| Login/logout/token/auth failures | `logsystem` | `auth_flow`, `failure_reason` (on failure) | `AUTH_LOGIN_FAILED` |
| Import/export/jobs/integration | `logsystem` | `batch_id`, `record_count`, `job_name` | `IMPORT_JOB_FINISHED` |
| Purge/archive/legal hold | `logsystem` | `archive_id`, `policy_name`, `approved_by` | `AUDIT_PURGE_EXECUTED` |
## 3) Canonical Schema (All Four Tables)
All four tables MUST implement the same logical columns. Physical PK name may vary (`LogPatientID`, `LogOrderID`, etc.).
### 3.1 Column contract
| Column | Type | Required | Max length | Description | Example |
| --- | --- | --- | --- | --- | --- |
| `LogID` (or table-specific PK) | `BIGINT UNSIGNED AUTO_INCREMENT` | Yes | N/A | Surrogate key per table | `987654` |
| `TblName` | `VARCHAR(64)` | Yes | 64 | Source business table | `patient` |
| `RecID` | `VARCHAR(64)` | Yes | 64 | Primary identifier of affected entity | `PAT000123` |
| `FldName` | `VARCHAR(128)` | Conditional | 128 | Changed field name, null for multi-field/bulk | `NameLast` |
| `FldValuePrev` | `TEXT` | Conditional | 65535 | Previous value (string or JSON) | `{"status":"PENDING"}` |
| `FldValueNew` | `TEXT` | Conditional | 65535 | New value (string or JSON) | `{"status":"VERIFIED"}` |
| `UserID` | `VARCHAR(64)` | Yes | 64 | Actor user id, or `SYSTEM` for non-user actions | `USR001` |
| `SiteID` | `VARCHAR(32)` | Yes | 32 | Facility/site context | `SITE01` |
| `DIDType` | `VARCHAR(32)` | No | 32 | Device identifier type | `UUID` |
| `DID` | `VARCHAR(128)` | No | 128 | Device identifier value | `6b8f...` |
| `MachineID` | `VARCHAR(128)` | No | 128 | Host/workstation identifier | `WS-LAB-07` |
| `SessionID` | `VARCHAR(128)` | Yes | 128 | Auth or workflow session identifier | `sess_abc123` |
| `AppID` | `VARCHAR(64)` | Yes | 64 | Calling client/application id | `clqms-api` |
| `ProcessID` | `VARCHAR(128)` | No | 128 | Process/workflow/job id | `job_20260325_01` |
| `WebPageID` | `VARCHAR(128)` | No | 128 | UI route/page id if user-driven | `patient-detail` |
| `EventID` | `VARCHAR(80)` | Yes | 80 | Canonical event code | `RESULT_RELEASED` |
| `ActivityID` | `VARCHAR(24)` | Yes | 24 | Canonical action enum | `UPDATE` |
| `Reason` | `VARCHAR(512)` | No | 512 | User/system reason or ticket reference | `Critical value corrected` |
| `LogDate` | `DATETIME(3)` | Yes | N/A | Event time in UTC | `2026-03-25 04:45:12.551` |
| `Context` | `JSON` (preferred) or `LONGTEXT` | Yes | N/A | Structured metadata payload | See section 5 |
| `IpAddress` | `VARCHAR(45)` | No | 45 | IPv4/IPv6 remote address | `10.10.2.44` |
### 3.2 Required/conditional rules
- `FldName`, `FldValuePrev`, and `FldValueNew` are required for single-field changes.
- For multi-field changes, set `FldName = NULL` and store a compact JSON diff under `Context.diff`.
- For non-mutating events (`READ`, `LOGIN`, `EXPORT`, `IMPORT`), `FldValuePrev` and `FldValueNew` may be null.
- `Context` is required for all rows. At minimum include `request_id` and `route` (or `job_name` for non-HTTP jobs).
## 4) DDL Template and Indexing
Use this template when creating a log table. Replace `${TABLE}` and `${PK}`.
```sql
CREATE TABLE `${TABLE}` (
`${PK}` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`TblName` VARCHAR(64) NOT NULL,
`RecID` VARCHAR(64) NOT NULL,
`FldName` VARCHAR(128) NULL,
`FldValuePrev` TEXT NULL,
`FldValueNew` TEXT NULL,
`UserID` VARCHAR(64) NOT NULL,
`SiteID` VARCHAR(32) NOT NULL,
`DIDType` VARCHAR(32) NULL,
`DID` VARCHAR(128) NULL,
`MachineID` VARCHAR(128) NULL,
`SessionID` VARCHAR(128) NOT NULL,
`AppID` VARCHAR(64) NOT NULL,
`ProcessID` VARCHAR(128) NULL,
`WebPageID` VARCHAR(128) NULL,
`EventID` VARCHAR(80) NOT NULL,
`ActivityID` VARCHAR(24) NOT NULL,
`Reason` VARCHAR(512) NULL,
`LogDate` DATETIME(3) NOT NULL,
`Context` JSON NOT NULL,
`IpAddress` VARCHAR(45) NULL,
PRIMARY KEY (`${PK}`),
INDEX `idx_${TABLE}_logdate` (`LogDate`),
INDEX `idx_${TABLE}_recid_logdate` (`RecID`, `LogDate`),
INDEX `idx_${TABLE}_userid_logdate` (`UserID`, `LogDate`),
INDEX `idx_${TABLE}_eventid_logdate` (`EventID`, `LogDate`),
INDEX `idx_${TABLE}_site_logdate` (`SiteID`, `LogDate`)
);
```
Optional JSON path index (DB engine specific):
- `Context.request_id`
- `Context.batch_id`
- `Context.job_name`
## 5) Context JSON Contract
`Context` MUST be valid JSON. Keep payload compact and predictable.
### 5.1 Required keys for all events
```json
{
"request_id": "a4f5b6c7",
"route": "PATCH /api/patient/123",
"timestamp_utc": "2026-03-25T04:45:12.551Z",
"entity_type": "patient",
"entity_version": 7
}
```
### 5.2 Additional keys by event class
- Patient/order/result mutation: `diff` (array of changed fields), `validation_profile`.
- Import/export/jobs: `batch_id`, `record_count`, `success_count`, `failure_count`, `job_name`.
- Auth/security events: `auth_flow`, `failure_reason`, `token_type` (never token value).
- Retention operations: `policy_name`, `archive_id`, `approved_by`, `window_start`, `window_end`.
### 5.3 Size and shape limits
- Maximum serialized `Context` size: 16 KB.
- `diff` array should include only audited fields, not entire entity snapshots.
- Store references (`file_id`, `blob_ref`) instead of large payloads.
## 6) Activity and Event Catalog Governance
`EventID` values MUST come from the ValueSet library, not hardcoded inline strings.
- Source file: `app/Libraries/Data/event_id.json`
- Runtime access: `\App\Libraries\ValueSet::getRaw('event_id')`
- Optional label lookup for reporting: `\App\Libraries\ValueSet::getLabel('event_id', $eventId)`
### 6.1 Allowed `ActivityID`
`CREATE`, `UPDATE`, `DELETE`, `READ`, `MERGE`, `SPLIT`, `CANCEL`, `REOPEN`, `VERIFY`, `AMEND`, `RETRACT`, `RELEASE`, `IMPORT`, `EXPORT`, `LOGIN`, `LOGOUT`, `LOCK`, `UNLOCK`, `RESET`
### 6.2 `EventID` naming pattern
- Format: `<DOMAIN>_<OBJECT>_<ACTION>`
- Character set: uppercase A-Z, numbers, underscore.
- Max length: 80.
- Examples: `PATIENT_DEMOGRAPHICS_UPDATED`, `ORDER_CANCELLED`, `AUTH_LOGIN_FAILED`.
### 6.3 Catalog lifecycle
- New `EventID` requires docs update and test coverage.
- New `EventID` must be added to `app/Libraries/Data/event_id.json` and deployed with cache refresh (`ValueSet::clearCache()`).
- Never repurpose an existing `EventID` to mean something else.
- Deprecated `EventID` remains queryable and documented for historical data.
## 7) Minimum Event Coverage (Must Implement)
### 7.1 `logpatient`
- `PATIENT_REGISTERED`, `PATIENT_DEMOGRAPHICS_UPDATED`, `PATIENT_MERGED`, `PATIENT_UNMERGED`
- `PATIENT_IDENTIFIER_UPDATED`, `PATIENT_CONSENT_UPDATED`, `PATIENT_INSURANCE_UPDATED`
- `VISIT_ADMITTED`, `VISIT_TRANSFERRED`, `VISIT_DISCHARGED`, `VISIT_STATUS_UPDATED`
### 7.2 `logorder`
- `ORDER_CREATED`, `ORDER_CANCELLED`, `ORDER_REOPENED`, `ORDER_TEST_ADDED`, `ORDER_TEST_REMOVED`
- `SPECIMEN_COLLECTED`, `SPECIMEN_RECEIVED`, `SPECIMEN_REJECTED`, `SPECIMEN_ALIQUOTED`, `SPECIMEN_DISPOSED`
- `RESULT_ENTERED`, `RESULT_UPDATED`, `RESULT_VERIFIED`, `RESULT_AMENDED`, `RESULT_RELEASED`, `RESULT_RETRACTED`, `RESULT_CORRECTED`
- `QC_RECORDED`, `QC_FAILED`, `QC_OVERRIDE_APPLIED`
### 7.3 `logmaster`
- `VALUESET_ITEM_CREATED`, `VALUESET_ITEM_UPDATED`, `VALUESET_ITEM_RETIRED`
- `TEST_DEFINITION_UPDATED`, `REFERENCE_RANGE_UPDATED`, `TEST_PANEL_MEMBERSHIP_UPDATED`
- `ANALYZER_CONFIG_UPDATED`, `INTEGRATION_CONFIG_UPDATED`, `CODING_SYSTEM_UPDATED`
- `USER_CREATED`, `USER_DISABLED`, `USER_PASSWORD_RESET`, `USER_ROLE_CHANGED`, `USER_PERMISSION_CHANGED`
- `SITE_CREATED`, `SITE_UPDATED`, `WORKSTATION_UPDATED`
### 7.4 `logsystem`
- `AUTH_LOGIN_SUCCESS`, `AUTH_LOGOUT_SUCCESS`, `AUTH_LOGIN_FAILED`, `AUTH_LOCKOUT_TRIGGERED`
- `TOKEN_ISSUED`, `TOKEN_REFRESHED`, `TOKEN_REVOKED`, `AUTHORIZATION_FAILED`
- `IMPORT_JOB_STARTED`, `IMPORT_JOB_FINISHED`, `EXPORT_JOB_STARTED`, `EXPORT_JOB_FINISHED`
- `JOB_STARTED`, `JOB_FINISHED`, `INTEGRATION_SYNC_STARTED`, `INTEGRATION_SYNC_FINISHED`
- `AUDIT_ARCHIVE_EXECUTED`, `AUDIT_PURGE_EXECUTED`, `LEGAL_HOLD_APPLIED`, `LEGAL_HOLD_RELEASED`
## 8) Capture Rules (Application Behavior)
### 8.1 Write timing
- For mutating transactions, write audit record in the same DB transaction where feasible.
- If asynchronous logging is required, enqueue within transaction and process with at-least-once delivery.
### 8.2 Failure policy
- Compliance-critical writes (patient, order, result, role/permission): fail request if audit write fails.
- Operational-only writes (non-critical job checkpoints): continue request, emit error log, retry in background.
- All audit write failures must produce `logsystem` event `AUDIT_WRITE_FAILED` with sanitized details.
### 8.3 Diff policy
- Single-field change: set `FldName`, `FldValuePrev`, `FldValueNew`.
- Multi-field change: set `FldName = NULL`, keep prev/new null or compact summary, place canonical diff in `Context.diff`.
- Bulk operations: include `batch_id`, `record_count`, sample `affected_ids` (capped), and source.
## 9) Security and Privacy Controls
### 9.1 Never log
- Passwords, raw JWTs, API secrets, private keys, OTP values.
- Full clinical free text unless explicitly required by policy.
### 9.2 Masking rules
- Identifiers with high sensitivity should be masked in `FldValuePrev/New` when not required.
- Token-like strings should be fully removed and replaced with `[REDACTED]`.
- Use deterministic masking where correlation is needed (e.g., hash + prefix).
### 9.3 Access control
- Insert permissions only for API/service accounts.
- No update/delete privileges for regular runtime users.
- Read access to logs is role-restricted and audited.
### 9.4 Tamper evidence
- Enable DB audit on DDL changes to log tables.
- Store periodic checksum snapshots of recent log ranges in secure storage.
- Record checksum run outcomes in `logsystem` (`AUDIT_CHECKSUM_CREATED`, `AUDIT_CHECKSUM_FAILED`).
## 10) Retention, Archive, and Purge
### 10.1 Default retention
- `logpatient`: 7 years
- `logorder`: 7 years
- `logmaster`: 5 years
- `logsystem`: 2 years
If regional policy requires longer periods, policy overrides these defaults.
### 10.2 Archive workflow
1. Select eligible rows by `LogDate` and legal-hold status.
2. Export to immutable archive format (compressed JSONL or parquet).
3. Verify checksums and row counts.
4. Write `AUDIT_ARCHIVE_EXECUTED` entry in `logsystem`.
### 10.3 Purge workflow
1. Require approval reference (`approved_by`, `change_ticket`).
2. Purge archived rows only.
3. Write `AUDIT_PURGE_EXECUTED` entry with table, date window, count, and archive reference.
## 11) Operational Monitoring
Track these SLIs/SLOs:
- Audit write success rate >= 99.9% for critical domains.
- P95 audit insert latency < 50 ms.
- Queue backlog age < 5 minutes (if async path is used).
- Zero unreviewed `AUDIT_WRITE_FAILED` older than 24 hours.
Alert on:
- Sustained write failures.
- Sudden drop in expected event volume.
- Purge/archive jobs without corresponding `logsystem` records.
## 12) Migration Strategy for Existing Logs
1. Inventory current columns and event vocabulary in all four tables.
2. Add missing canonical columns with nullable defaults.
3. Backfill required values (`AppID`, `SessionID`, `Context` minimum keys) where derivable.
4. Introduce canonical `EventID` mapping table for legacy names.
5. Enforce NOT NULL constraints only after backfill validation succeeds.
## 13) Testing Requirements
### 13.1 Automated tests
- Feature tests for representative endpoints must assert audit row creation.
- Assert table target, `ActivityID`, `EventID`, `RecID`, and required `Context` keys.
- Assert `EventID` exists in `\App\Libraries\ValueSet::getRaw('event_id')`.
- Add negative tests for audit failure policy (critical path blocks, non-critical path retries).
### 13.2 Test matrix minimum
- One success and one failure scenario per major domain (`patient`, `order`, `master`, `system`).
- One bulk operation scenario validating `batch_id` and counts.
- One security scenario validating redaction of sensitive fields.
## 14) Implementation Checklist (Phased)
### Phase 1 - Schema and constants
1. Create/align all four log tables to canonical schema.
2. Add shared enums/constants for `ActivityID` and `EventID`.
3. Add and maintain `app/Libraries/Data/event_id.json` as the `EventID` source of truth.
4. Add DB indexes listed in section 4.
### Phase 2 - Audit service
1. Implement centralized audit writer service.
2. Add helpers to normalize actor/device/session/context.
3. Add diff builder utility for single and multi-field changes.
### Phase 3 - Instrumentation
1. Instrument patient and order flows first (compliance-critical).
2. Instrument master and system flows.
3. Add fallback/retry path and `AUDIT_WRITE_FAILED` emission.
### Phase 4 - Validation and rollout
1. Add feature tests and failure-path tests.
2. Validate dashboards/queries for each table.
3. Release with runbook updates and retention job schedule.
## 15) Acceptance Criteria
The implementation is complete when all statements below are true:
- Every protected endpoint emits at least one canonical audit row.
- Each row has valid `ActivityID`, `EventID` (present in ValueSet `event_id`), `LogDate` (UTC), and non-empty `Context` with required keys.
- Sensitive values are redacted/masked per section 9.
- Archive and purge operations are fully traceable in `logsystem`.
- Tests cover critical success/failure paths and pass in CI.

View File

@ -1,337 +0,0 @@
# Calculator Service Operators Reference
## Overview
The `CalculatorService` (`app/Services/CalculatorService.php`) evaluates formulas with Symfony's `ExpressionLanguage`. This document lists the operators, functions, and constants that are available in the current implementation.
---
## API Endpoints
All endpoints live under `/api` and accept JSON. Responses use the standard `{ status, message, data }` envelope unless stated otherwise.
### Calculate By Test Site
Uses the `testdefcal` definition for a test site. The incoming body supplies the variables required by the formula.
```http
POST /api/calc/testsite/123
Content-Type: application/json
{
"result": 85,
"gender": "female",
"age": 30
}
```
Response:
```json
{
"status": "success",
"data": {
"result": 92.4,
"testSiteID": 123,
"formula": "{result} * {factor} + {age}",
"variables": {
"result": 85,
"gender": "female",
"age": 30
}
}
}
```
### Calculate By Code Or Name
Evaluates a configured calculation by `TestSiteCode` or `TestSiteName`. Returns a compact map with a single key/value or `{}` on failure.
```http
POST /api/calc/testcode/GLU
Content-Type: application/json
{
"result": 110,
"factor": 1.1
}
```
Response:
```json
{
"GLU": 121
}
```
---
## Supported Operators
### Arithmetic Operators
| Operator | Description | Example | Result |
|----------|-------------|---------|--------|
| `+` | Addition | `5 + 3` | `8` |
| `-` | Subtraction | `10 - 4` | `6` |
| `*` | Multiplication | `6 * 7` | `42` |
| `/` | Division | `20 / 4` | `5` |
| `%` | Modulo | `20 % 6` | `2` |
| `**` | Exponentiation (power) | `2 ** 3` | `8` |
### Comparison Operators
| Operator | Description | Example |
|----------|-------------|---------|
| `==` | Equal | `{result} == 10` |
| `!=` | Not equal | `{result} != 10` |
| `<` | Less than | `{result} < 10` |
| `<=` | Less than or equal | `{result} <= 10` |
| `>` | Greater than | `{result} > 10` |
| `>=` | Greater than or equal | `{result} >= 10` |
### Logical Operators
| Operator | Description | Example |
|----------|-------------|---------|
| `and` / `&&` | Logical AND | `{result} > 0 and {factor} > 0` |
| `or` / `||` | Logical OR | `{gender} == 1 or {gender} == 2` |
| `!` / `not` | Logical NOT | `not ({result} > 0)` |
### Conditional Operators
| Operator | Description | Example |
|----------|-------------|---------|
| `?:` | Ternary | `{result} > 10 ? {result} : 10` |
| `??` | Null coalescing | `{result} ?? 0` |
### Parentheses
Use parentheses to control operation precedence:
```
(2 + 3) * 4 // Result: 20
2 + 3 * 4 // Result: 14
```
### Notes
- `^` is bitwise XOR (not exponentiation). Use `**` for powers.
- Variables must be numeric after normalization (gender is mapped to 0/1/2).
---
## Functions
Only the default ExpressionLanguage functions are available:
| Function | Description | Example |
|----------|-------------|---------|
| `min(a, b, ...)` | Minimum value | `min({result}, 10)` |
| `max(a, b, ...)` | Maximum value | `max({result}, 10)` |
| `constant(name)` | PHP constant by name | `constant("PHP_INT_MAX")` |
| `enum(name)` | PHP enum case by name | `enum("App\\Enum\\Status::Active")` |
---
## Constants
ExpressionLanguage recognizes boolean and null literals:
| Constant | Value | Description |
|----------|-------|-------------|
| `true` | `true` | Boolean true |
| `false` | `false` | Boolean false |
| `null` | `null` | Null value |
---
## Variables in CalculatorService
When using `calculateFromDefinition()`, the following variables are automatically available:
| Variable | Description | Type |
|----------|-------------|------|
| `{result}` | The test result value | Float |
| `{factor}` | Calculation factor (default: 1) | Float |
| `{gender}` | Gender value (0=Unknown, 1=Female, 2=Male) | Integer |
| `{age}` | Patient age | Float |
| `{ref_low}` | Reference range low value | Float |
| `{ref_high}` | Reference range high value | Float |
### Gender Mapping
The `gender` variable accepts the following values:
| Value | Description |
|-------|-------------|
| `0` | Unknown |
| `1` | Female |
| `2` | Male |
Or use string values: `'unknown'`, `'female'`, `'male'`
---
## Implicit Multiplication
Implicit multiplication is not supported. Always use `*` between values:
| Expression | Use Instead |
|------------|-------------|
| `2x` | `2 * x` |
| `{result}{factor}` | `{result} * {factor}` |
---
## Usage Examples
### Basic Calculation
```php
use App\Services\CalculatorService;
$calculator = new CalculatorService();
// Simple arithmetic
$result = $calculator->calculate("5 + 3 * 2");
// Result: 11
// Using min/max
$result = $calculator->calculate("max({result}, 10)", ['result' => 7]);
// Result: 10
```
### With Variables
```php
$formula = "{result} * {factor} + 10";
$variables = [
'result' => 5.2,
'factor' => 2
];
$result = $calculator->calculate($formula, $variables);
// Result: 20.4
```
### BMI Calculation
```php
$formula = "{weight} / ({height} ** 2)";
$variables = [
'weight' => 70, // kg
'height' => 1.75 // meters
];
$result = $calculator->calculate($formula, $variables);
// Result: 22.86
```
### Gender-Based Calculation
```php
// Apply different multipliers based on gender
$formula = "{result} * (1 + 0.1 * {gender})";
$variables = [
'result' => 100,
'gender' => 1 // Female = 1
];
$result = $calculator->calculate($formula, $variables);
// Result: 110
```
### Complex Formula
```php
// Pythagorean theorem
$formula = "(({a} ** 2 + {b} ** 2) ** 0.5)";
$variables = [
'a' => 3,
'b' => 4
];
$result = $calculator->calculate($formula, $variables);
// Result: 5
```
### Using calculateFromDefinition
```php
$calcDef = [
'FormulaCode' => '{result} * {factor} + {gender}',
'Factor' => 2
];
$testValues = [
'result' => 10,
'gender' => 1 // Female
];
$result = $calculator->calculateFromDefinition($calcDef, $testValues);
// Result: 21 (10 * 2 + 1)
```
---
## Formula Validation
Validate formulas before storing them:
```php
$validation = $calculator->validate("{result} / {factor}");
// Returns: ['valid' => true, 'error' => null]
$validation = $calculator->validate("{result} /");
// Returns: ['valid' => false, 'error' => 'Error message']
```
### Extract Variables
Get a list of variables used in a formula:
```php
$variables = $calculator->extractVariables("{result} * {factor} + {age}");
// Returns: ['result', 'factor', 'age']
```
---
## Error Handling
The service throws exceptions for invalid formulas or missing variables:
```php
try {
$result = $calculator->calculate("{result} / 0");
} catch (\Exception $e) {
// Handle division by zero or other errors
log_message('error', $e->getMessage());
}
```
Common errors:
- **Invalid formula syntax**: Malformed expressions
- **Missing variable**: Variable placeholder not provided in data array
- **Non-numeric value**: Variables must be numeric
- **Division by zero**: Mathematical error
---
## Best Practices
1. **Always validate formulas** before storing in database
2. **Use placeholder syntax** `{variable_name}` for clarity
3. **Handle exceptions** in production code
4. **Test edge cases** like zero values and boundary conditions
5. **Document formulas** with comments in your code
---
## References
- [Symfony ExpressionLanguage](https://symfony.com/doc/current/components/expression_language.html)
- `app/Services/CalculatorService.php`

View File

@ -1,421 +0,0 @@
# Test Rule Engine Documentation
## Overview
The CLQMS Rule Engine evaluates business rules that inspect orders, patients, and tests, then executes actions when the compiled condition matches.
Rules are authored using a domain specific language stored in `ruledef.ConditionExpr`. Before the platform executes any rule, the DSL must be compiled into JSON and stored in `ConditionExprCompiled`, and each rule must be linked to the tests it should influence via `testrule`.
### Execution Flow
1. Write or edit the DSL in `ConditionExpr`.
2. POST the expression to `POST /api/rule/compile` to validate syntax and produce compiled JSON.
3. Save the compiled payload into `ConditionExprCompiled` and persist the rule in `ruledef`.
4. Link the rule to one or more tests through `testrule.TestSiteID` (rules only run for linked tests).
5. When the configured event fires (`test_created` or `result_updated`), the engine evaluates `ConditionExprCompiled` and runs the resulting `then` or `else` actions.
> **Note:** The rule engine currently fires only for `test_created` and `result_updated`. Other event codes can exist in the database but are not triggered by the application unless additional `RuleEngineService::run(...)` calls are added.
## Event Triggers
| Event Code | Status | Trigger Point |
|------------|--------|----------------|
| `test_created` | Active | Fired after a new test row is persisted; the handler calls `RuleEngineService::run('test_created', ...)` to evaluate test-scoped rules |
| `result_updated` | Active | Fired whenever a test result is saved or updated so result-dependent rules run immediately |
Other event codes remain in the database for future workflows, but only `test_created` and `result_updated` are executed by the current application flow.
## Rule Structure
```
Rule
├── Event Trigger (when to run)
├── Conditions (when to match)
└── Actions (what to do)
```
The DSL expression lives in `ConditionExpr`. The compile endpoint (`/api/rule/compile`) renders the lifeblood of execution, producing `conditionExpr`, `valueExpr`, `then`, and `else` nodes that the engine consumes at runtime.
## Syntax Guide
### Basic Format
```
if(condition; then-action; else-action)
```
### Logical Operators
- Use `&&` for AND (all sub-conditions must match).
- Use `||` for OR (any matching branch satisfies the rule).
- Surround mixed logic with parentheses for clarity and precedence.
### Multi-Action Syntax
Actions within any branch are separated by `:` and evaluated in order. Every `then` and `else` branch must end with an action; use `nothing` when no further work is required.
```
if(sex('M'); result_set(0.5):test_insert('HBA1C'); nothing)
```
### Multiple Rules
Create each rule as its own `ruledef` row; do not chain expressions with commas. The `testrule` table manages rule-to-test mappings, so multiple rules can attach to the same test. Example:
1. Insert `RULE_MALE_RESULT` and `RULE_SENIOR_COMMENT` in `ruledef`.
2. Add two `testrule` rows linking each rule to the appropriate `TestSiteID`.
Each rule compiles and runs independently when its trigger fires and the test is linked.
## Available Functions
### Conditions
| Function | Description | Example |
|----------|-------------|---------|
| `sex('M'|'F')` | Match patient sex | `sex('M')` |
| `priority('R'|'S'|'U')` | Match order priority | `priority('S')` |
| `age > 18` | Numeric age comparisons (`>`, `<`, `>=`, `<=`) | `age >= 18 && age <= 65` |
| `requested('CODE')` | Check whether the order already requested a test (queries `patres`) | `requested('GLU')` |
### Logical Operators
| Operator | Meaning | Example |
|----------|---------|---------|
| `&&` | AND (all truthy) | `sex('M') && age > 40` |
| `||` | OR (any truthy) | `sex('M') || age > 65` |
| `()` | Group expressions | `(sex('M') && age > 40) || priority('S')` |
## Actions
| Action | Description | Example |
|--------|-------------|---------|
| `result_set(value)` | (deprecated) Write to `patres.Result` for the current context test | `result_set(0.5)` |
| `result_set('CODE', value)` | Target a specific test by `TestSiteCode`, allowing multiple tests to be updated in one rule | `result_set('tesA', 0.5)` |
| `test_insert('CODE')` | Insert a test row by `TestSiteCode` if it doesnt already exist for the order | `test_insert('HBA1C')` |
| `test_delete('CODE')` | Remove a previously requested test from the current order when the rule deems it unnecessary | `test_delete('INS')` |
| `comment_insert('text')` | Insert an order comment (`ordercom`) describing priority or clinical guidance | `comment_insert('Male patient - review')` |
| `nothing` | Explicit no-op to terminate an action chain | `nothing` |
> **Note:** `set_priority()` was removed. Use `comment_insert()` for priority notes without altering billing.
## Runtime Requirements
1. **Compiled expression required:** Rules without `ConditionExprCompiled` are ignored (see `RuleEngineService::run`).
2. **Order context:** `context.order.InternalOID` must exist for any action that writes to `patres` or `ordercom`.
3. **TestSiteID:** `result_set()` needs `testSiteID` (either provided in context or from `order.TestSiteID`). When you provide a `TestSiteCode` as the first argument (`result_set('tesA', value)`), the engine resolves that code before writing the result. `test_insert()` resolves a `TestSiteID` via the `TestSiteCode` in `TestDefSiteModel`, and `test_delete()` removes the matching `TestSiteID` rows when needed.
4. **Requested check:** `requested('CODE')` inspects `patres` rows for the same `OrderID` and `TestSiteCode`.
## Examples
```
if(sex('M'); result_set('tesA', 0.5):result_set('tesB', 1.2); result_set('tesA', 0.6):result_set('tesB', 1.0))
```
Sets both `tesA`/`tesB` results together per branch.
```
if(requested('GLU'); test_insert('HBA1C'):test_insert('INS'); nothing)
```
Adds new tests when glucose is already requested.
```
if(sex('M') && age > 40; result_set(1.2); result_set(1.0))
```
```
if((sex('M') && age > 40) || (sex('F') && age > 50); result_set(1.5); result_set(1.0))
```
```
if(priority('S'); result_set('URGENT'):test_insert('STAT_TEST'); result_set('NORMAL'))
```
```
if(sex('M') && age > 40; result_set(1.5):test_insert('EXTRA_TEST'):comment_insert('Male over 40'); nothing)
```
```
if(sex('F') && (age >= 18 && age <= 50) && priority('S'); result_set('HIGH_PRIO'):comment_insert('Female stat 18-50'); result_set('NORMAL'))
```
```
if(requested('GLU'); test_delete('INS'):comment_insert('Duplicate insulin request removed'); nothing)
```
## API Endpoints
All endpoints live under `/api/rule` (singular) and accept JSON. Responses use the standard `{ status, message, data }` envelope. These endpoints require auth (bearer token).
### List Rules
```http
GET /api/rule?EventCode=test_created&TestSiteID=12&search=glucose
```
Query Params:
- `EventCode` (optional) filter by event code.
- `TestSiteID` (optional) filter rules linked to a test site.
- `search` (optional) partial match against `RuleName`.
Response:
```json
{
"status": "success",
"message": "fetch success",
"data": [
{
"RuleID": 1,
"RuleCode": "RULE_001",
"RuleName": "Sex-based result",
"EventCode": "test_created",
"ConditionExpr": "if(sex('M'); result_set(0.5); result_set(0.6))",
"ConditionExprCompiled": "{...}"
}
]
}
```
### Get Rule
```http
GET /api/rule/1
```
Response includes `linkedTests`:
```json
{
"status": "success",
"message": "fetch success",
"data": {
"RuleID": 1,
"RuleCode": "RULE_001",
"RuleName": "Sex-based result",
"EventCode": "test_created",
"ConditionExpr": "if(sex('M'); result_set(0.5); result_set(0.6))",
"ConditionExprCompiled": "{...}",
"linkedTests": [1, 2]
}
}
```
### Create Rule
```http
POST /api/rule
Content-Type: application/json
{
"RuleCode": "RULE_001",
"RuleName": "Sex-based result",
"EventCode": "test_created",
"ConditionExpr": "if(sex('M'); result_set(0.5); result_set(0.6))",
"ConditionExprCompiled": "<compiled JSON here>",
"TestSiteIDs": [1, 2]
}
```
Response:
```json
{
"status": "success",
"message": "Rule created successfully",
"data": {
"RuleID": 1
}
}
```
### Update Rule
```http
PATCH /api/rule/1
Content-Type: application/json
{
"RuleName": "Sex-based result v2",
"ConditionExpr": "if(sex('M'); result_set(0.7); result_set(0.6))",
"ConditionExprCompiled": "<compiled JSON here>",
"TestSiteIDs": [1, 3]
}
```
Response:
```json
{
"status": "success",
"message": "Rule updated successfully",
"data": {
"RuleID": 1
}
}
```
### Delete Rule
```http
DELETE /api/rule/1
```
Response:
```json
{
"status": "success",
"message": "Rule deleted successfully",
"data": {
"RuleID": 1
}
}
```
### Compile DSL
Validates the DSL and returns a compiled JSON structure that should be persisted in `ConditionExprCompiled`.
```http
POST /api/rule/compile
Content-Type: application/json
{
"expr": "if(sex('M'); result_set(0.5); result_set(0.6))"
}
```
Response:
```json
{
"status": "success",
"data": {
"raw": "if(sex('M'); result_set(0.5); result_set(0.6))",
"compiled": {
"conditionExpr": "sex('M')",
"then": ["result_set(0.5)"],
"else": ["result_set(0.6)"]
},
"conditionExprCompiled": "{...}"
}
}
```
### Evaluate Expression (Validation)
This endpoint evaluates an expression against a runtime context. It does not compile DSL or persist the result.
```http
POST /api/rule/validate
Content-Type: application/json
{
"expr": "order[\"Age\"] > 18",
"context": {
"order": {
"Age": 25
}
}
}
```
Response:
```json
{
"status": "success",
"data": {
"valid": true,
"result": true
}
}
```
## Database Schema
### Tables
- **ruledef** stores rule metadata, raw DSL, and compiled JSON.
- **testrule** mapping table that links rules to tests via `TestSiteID`.
- **ruleaction** deprecated. Actions are now embedded in `ConditionExprCompiled`.
### Key Columns
| Column | Table | Description |
|--------|-------|-------------|
| `EventCode` | ruledef | The trigger event (typically `test_created` or `result_updated`). |
| `ConditionExpr` | ruledef | Raw DSL expression (semicolon syntax). |
| `ConditionExprCompiled` | ruledef | JSON payload consumed at runtime (`then`, `else`, etc.). |
| `ActionType` / `ActionParams` | ruleaction | Deprecated; actions live in compiled JSON now. |
## Best Practices
1. Always run `POST /api/rule/compile` before persisting a rule so `ConditionExprCompiled` exists.
2. Link each rule to the relevant tests via `testrule.TestSiteID`—rules are scoped to linked tests.
3. Use multi-action (`:`) to bundle several actions in a single branch; finish the branch with `nothing` if no further work is needed.
4. Prefer `comment_insert()` over the removed `set_priority()` action when documenting priority decisions.
5. Group complex boolean logic with parentheses for clarity when mixing `&&` and `||`.
6. Use `requested('CODE')` responsibly; it performs a database lookup on `patres` so avoid invoking it in high-frequency loops without reason.
## Migration Guide
### Syntax Changes (v2.0)
The DSL moved from ternary (`condition ? action : action`) to semicolon syntax. Existing rules must be migrated via the provided script.
| Old Syntax | New Syntax |
|------------|------------|
| `if(condition ? action : action)` | `if(condition; action; action)` |
#### Migration Examples
```
# BEFORE
if(sex('M') ? result_set(0.5) : result_set(0.6))
# AFTER
if(sex('M'); result_set(0.5); result_set(0.6))
```
```
# BEFORE
if(sex('F') ? set_priority('S') : nothing)
# AFTER
if(sex('F'); comment_insert('Female patient - review priority'); nothing)
```
#### Migration Process
Run the migration which:
1. Converts ternary syntax to semicolon syntax.
2. Recompiles every expression into JSON so the engine consumes `ConditionExprCompiled` directly.
3. Eliminates reliance on the `ruleaction` table.
```bash
php spark migrate
```
## Troubleshooting
### Rule Not Executing
1. Ensure the rule has a compiled payload (`ConditionExprCompiled`).
2. Confirm the rule is linked to the relevant `TestSiteID` in `testrule`.
3. Verify the `EventCode` matches the currently triggered event (`test_created` or `result_updated`).
4. Check that `EndDate IS NULL` for both `ruledef` and `testrule` (soft deletes disable execution).
5. Use `/api/rule/compile` to validate the DSL and view errors.
### Invalid Expression
1. POST the expression to `/api/rule/compile` to get a detailed compilation error.
2. If using `/api/rule/validate`, supply the expected `context` payload; the endpoint simply evaluates the expression without saving it.
### Runtime Errors
- `RESULT_SET requires context.order.InternalOID` or `testSiteID`: include those fields in the context passed to `RuleEngineService::run()`.
- `TEST_INSERT` failures mean the provided `TestSiteCode` does not exist or the rule attempted to insert a duplicate test; check `testdefsite` and existing `patres` rows.
- `COMMENT_INSERT requires comment`: ensure the action provides text.

View File

@ -4000,9 +4000,6 @@ paths:
schema:
type: object
properties:
TestCode:
type: string
description: Test Code (required) - maps to HostTestCode or ClientTestCode
HostType:
type: string
description: Host type code
@ -4031,8 +4028,6 @@ paths:
type: string
ClientTestName:
type: string
required:
- TestCode
responses:
'201':
description: Test mapping created
@ -4135,9 +4130,6 @@ paths:
schema:
type: object
properties:
TestCode:
type: string
description: Test Code - maps to HostTestCode or ClientTestCode
HostType:
type: string
HostID:

View File

@ -45,13 +45,10 @@
requestBody:
required: true
content:
application/json:
schema:
type: object
application/json:
schema:
type: object
properties:
TestCode:
type: string
description: Test Code (required) - maps to HostTestCode or ClientTestCode
HostType:
type: string
description: Host type code
@ -80,8 +77,6 @@
type: string
ClientTestName:
type: string
required:
- TestCode
responses:
'201':
description: Test mapping created
@ -185,9 +180,6 @@
schema:
type: object
properties:
TestCode:
type: string
description: Test Code - maps to HostTestCode or ClientTestCode
HostType:
type: string
HostID:

View File

@ -290,9 +290,7 @@ class TestCreateVariantsTest extends CIUnitTestCase
$show->assertStatus(200);
$data = json_decode($show->getJSON(), true)['data'];
$this->assertNotEmpty($data['testmap']);
$this->assertSame('HIS', $data['testmap'][0]['HostType']);
$this->assertSame('SITE', $data['testmap'][0]['ClientType']);
$this->assertArrayNotHasKey('testmap', $data);
}
public function testPatchTechnicalWithFlatTestMapPayload(): void
@ -324,9 +322,7 @@ class TestCreateVariantsTest extends CIUnitTestCase
$show->assertStatus(200);
$data = json_decode($show->getJSON(), true)['data'];
$this->assertNotEmpty($data['testmap']);
$this->assertSame('HIS', $data['testmap'][0]['HostType']);
$this->assertSame('SITE', $data['testmap'][0]['ClientType']);
$this->assertArrayNotHasKey('testmap', $data);
}
public function testCreateCalculatedTestWithoutReferenceOrMap(): void