Release for qwen3:4b model

2025-08-01 18:15:57 +02:00 · 2025-08-01 18:15:57 +02:00 · d1fa9385e7
commit d1fa9385e7
parent 6f5e817011
16 changed files with 630 additions and 41 deletions
--- a/.env
+++ b/.env
@ -1,20 +1,15 @@
 # Ollama configuration
-# LLM_OLLAMA_BASE_URL=http://192.168.0.140:11434
-# LLM_OLLAMA_BASE_URL=http://192.168.0.122:11434
-LLM_OLLAMA_BASE_URL="https://api-amer-sandbox-gbl-mdm-hub.pfizer.com/ollama"
-LLM_OLLAMA_MODEL=phi4-mini:latest
-# LLM_OLLAMA_MODEL=smollm:360m
-# LLM_OLLAMA_MODEL=qwen3:0.6b
-# LLM_OLLAMA_MODEL=qwen3:1.7b
-# LLM_OLLAMA_MODEL=qwen3:8b
-# Logging configuration
+# LLM_OLLAMA_BASE_URL="https://api-amer-sandbox-gbl-mdm-hub.pfizer.com/ollama"
+LLM_OLLAMA_BASE_URL=http://ollama-jira:11434
+# LLM_OLLAMA_MODEL=phi4-mini:latest
+LLM_OLLAMA_MODEL=qwen3:4b
 LOG_LEVEL=DEBUG
 # Ollama API Key (required when using Ollama mode)
 # Langfuse configuration
 LANGFUSE_ENABLED=false
-LANGFUSE_PUBLIC_KEY="pk-lf-"
-LANGFUSE_SECRET_KEY="sk-lf-"
-LANGFUSE_HOST="https://cloud.langfuse.com"
+# LANGFUSE_PUBLIC_KEY="pk-lf-"
+# LANGFUSE_SECRET_KEY="sk-lf-"
+# LANGFUSE_HOST="https://cloud.langfuse.com"
 # Gemini configuration
 LLM_GEMINI_API_KEY=""
 LLM_GEMINI_MODEL="gemini-2.5-flash"
--- a/.env-cnf_local
+++ b/.env-cnf_local
@ -0,0 +1,18 @@
+# Ollama configuration
+# LLM_OLLAMA_BASE_URL=http://192.168.0.140:11434
+LLM_OLLAMA_BASE_URL=http://192.168.0.122:11434
+# LLM_OLLAMA_BASE_URL="https://api-amer-sandbox-gbl-mdm-hub.pfizer.com/ollama"
+# LLM_OLLAMA_MODEL=phi4-mini:latest
+LLM_OLLAMA_MODEL=qwen3:4b
+# Logging configuration
+LOG_LEVEL=DEBUG
+# Ollama API Key (required when using Ollama mode)
+# Langfuse configuration
+LANGFUSE_ENABLED=true
+LANGFUSE_PUBLIC_KEY="pk-lf-38238bc3-ffa1-48d3-8c8f-2048ecf8cc54"
+LANGFUSE_SECRET_KEY="sk-lf-36f6bd37-b0e2-4656-9c63-fe98915e8872"
+LANGFUSE_HOST="http://192.168.0.122:3000"
+# Gemini configuration
+LLM_GEMINI_API_KEY=""
+LLM_GEMINI_MODEL="gemini-2.5-flash"
+LLM_MODE=ollama
--- a/.env_cnf_amer
+++ b/.env_cnf_amer
@ -0,0 +1,16 @@
+# Ollama configuration
+# LLM_OLLAMA_BASE_URL="https://api-amer-sandbox-gbl-mdm-hub.pfizer.com/ollama"
+LLM_OLLAMA_BASE_URL=http://ollama-jira:11434
+# LLM_OLLAMA_MODEL=phi4-mini:latest
+LLM_OLLAMA_MODEL=qwen3:4b
+LOG_LEVEL=DEBUG
+# Ollama API Key (required when using Ollama mode)
+# Langfuse configuration
+LANGFUSE_ENABLED=false
+# LANGFUSE_PUBLIC_KEY="pk-lf-"
+# LANGFUSE_SECRET_KEY="sk-lf-"
+# LANGFUSE_HOST="https://cloud.langfuse.com"
+# Gemini configuration
+LLM_GEMINI_API_KEY=""
+LLM_GEMINI_MODEL="gemini-2.5-flash"
+LLM_MODE=ollama
--- a/app/handlers.py
+++ b/app/handlers.py
@ -28,6 +28,12 @@ class GetResponseRequest(BaseModel):
@jira_router.post("/getResponse")
 async def get_jira_response(request: GetResponseRequest):
    """Get response attribute provided by ollama for a given issueKey."""
+    if requests_queue.are_there_any_open_requests_for_issue_key(request.issueKey):
+        raise HTTPException(
+            status_code=409,
+            detail=f"There are still pending or processing requests for issueKey: {request.issueKey}. Please wait for them to be processed."
+        )
+
    matched_request = requests_queue.get_latest_completed_by_issue_key(request.issueKey)
    if not matched_request:
        raise HTTPException(status_code=404, detail=f"No completed request found for issueKey: {request.issueKey}")
--- a/config/application-cnf_amer.yml
+++ b/config/application-cnf_amer.yml
@ -0,0 +1,77 @@
+# Default application configuration
+llm:
+  # The mode to run the application in.
+  # Can be 'openai' or 'ollama'.
+  # This can be overridden by the LLM_MODE environment variable.
+  mode: ollama # Change mode to gemini
+
+  # Settings for OpenAI-compatible APIs (like OpenRouter)
+  openai:
+    # It's HIGHLY recommended to set this via an environment variable
+    # instead of saving it in this file.
+    # Can be overridden by OPENAI_API_KEY
+    # api_key: "sk-or-v1-..."
+    # api_key: "your-openai-api-key" # Keep this commented out or set to a placeholder
+
+    # Can be overridden by OPENAI_API_BASE_URL
+    # api_base_url: "https://openrouter.ai/api/v1"
+    # api_base_url: "https://api.openai.com/v1" # Remove or comment out this line
+
+    # Can be overridden by OPENAI_MODEL
+    # model: "deepseek/deepseek-chat:free"
+    # model: "gpt-4o" # Keep this commented out or set to a placeholder
+
+  # Settings for Gemini
+  gemini:
+    # It's HIGHLY recommended to set this via an environment variable
+    # instead of saving it in this file.
+    # Can be overridden by GEMINI_API_KEY
+    api_key: "" 
+
+    # Can be overridden by GEMINI_MODEL
+    # model: "gemini-2.5-flash" 
+    model: "gemini-2.5-flash-lite-preview-06-17"
+
+    # Can be overridden by GEMINI_API_BASE_URL
+    api_base_url: "https://generativelanguage.googleapis.com/v1beta/" # Add for Gemini
+
+  # Settings for Ollama
+  ollama:
+    # Can be overridden by OLLAMA_BASE_URL
+    base_url: "http://ollama-jira:11434"
+    # base_url: "https://api-amer-sandbox-gbl-mdm-hub.pfizer.com/ollama"
+
+
+    # Can be overridden by OLLAMA_MODEL
+    # model: "phi4-mini:latest"
+    # model: "qwen3:1.7b"
+    # model: "smollm:360m"
+    # model: "qwen3:0.6b"
+    model: "qwen3:4b"
+# Langfuse configuration for observability and analytics
+langfuse:
+  # Enable or disable Langfuse integration
+  # Can be overridden by LANGFUSE_ENABLED environment variable
+  enabled: false
+  
+  # Langfuse API credentials
+  # It's HIGHLY recommended to set these via environment variables
+  # instead of saving them in this file
+  public_key: "pk-lf-"
+  secret_key: "sk-lf-"
+  # host: "https://cloud.langfuse.com"
+  # host: "http://192.168.0.122:3000"
+
+# Processor configuration
+processor:
+ # Interval in seconds between polling for new Jira analysis requests
+ # Can be overridden by PROCESSOR_POLL_INTERVAL_SECONDS environment variable
+ poll_interval_seconds: 10
+ 
+ # Maximum number of retries for failed Jira analysis requests
+ # Can be overridden by PROCESSOR_MAX_RETRIES environment variable
+ max_retries: 0
+ 
+ # Initial delay in seconds before the first retry attempt (exponential backoff)
+ # Can be overridden by PROCESSOR_INITIAL_RETRY_DELAY_SECONDS environment variable
+ initial_retry_delay_seconds: 60
--- a/config/application-cnf_local.yml
+++ b/config/application-cnf_local.yml
@ -0,0 +1,77 @@
+# Default application configuration
+llm:
+  # The mode to run the application in.
+  # Can be 'openai' or 'ollama'.
+  # This can be overridden by the LLM_MODE environment variable.
+  mode: ollama # Change mode to gemini
+
+  # Settings for OpenAI-compatible APIs (like OpenRouter)
+  openai:
+    # It's HIGHLY recommended to set this via an environment variable
+    # instead of saving it in this file.
+    # Can be overridden by OPENAI_API_KEY
+    # api_key: "sk-or-v1-..."
+    # api_key: "your-openai-api-key" # Keep this commented out or set to a placeholder
+
+    # Can be overridden by OPENAI_API_BASE_URL
+    # api_base_url: "https://openrouter.ai/api/v1"
+    # api_base_url: "https://api.openai.com/v1" # Remove or comment out this line
+
+    # Can be overridden by OPENAI_MODEL
+    # model: "deepseek/deepseek-chat:free"
+    # model: "gpt-4o" # Keep this commented out or set to a placeholder
+
+  # Settings for Gemini
+  gemini:
+    # It's HIGHLY recommended to set this via an environment variable
+    # instead of saving it in this file.
+    # Can be overridden by GEMINI_API_KEY
+    api_key: "" 
+
+    # Can be overridden by GEMINI_MODEL
+    # model: "gemini-2.5-flash" 
+    model: "gemini-2.5-flash-lite-preview-06-17"
+
+    # Can be overridden by GEMINI_API_BASE_URL
+    api_base_url: "https://generativelanguage.googleapis.com/v1beta/" # Add for Gemini
+
+  # Settings for Ollama
+  ollama:
+    # Can be overridden by OLLAMA_BASE_URL
+    base_url: "http://192.168.0.122:11434"
+    # base_url: "https://api-amer-sandbox-gbl-mdm-hub.pfizer.com/ollama"
+
+
+    # Can be overridden by OLLAMA_MODEL
+    # model: "phi4-mini:latest"
+    # model: "qwen3:1.7b"
+    # model: "smollm:360m"
+    # model: "qwen3:0.6b"
+    model: "qwen3:4b"
+# Langfuse configuration for observability and analytics
+langfuse:
+  # Enable or disable Langfuse integration
+  # Can be overridden by LANGFUSE_ENABLED environment variable
+  enabled: true
+  
+  # Langfuse API credentials
+  # It's HIGHLY recommended to set these via environment variables
+  # instead of saving them in this file
+  public_key: "pk-lf-38238bc3-ffa1-48d3-8c8f-2048ecf8cc54"
+  secret_key: "sk-lf-36f6bd37-b0e2-4656-9c63-fe98915e8872"
+  # host: "https://cloud.langfuse.com"
+  host: "http://192.168.0.122:3000"
+
+# Processor configuration
+processor:
+ # Interval in seconds between polling for new Jira analysis requests
+ # Can be overridden by PROCESSOR_POLL_INTERVAL_SECONDS environment variable
+ poll_interval_seconds: 10
+ 
+ # Maximum number of retries for failed Jira analysis requests
+ # Can be overridden by PROCESSOR_MAX_RETRIES environment variable
+ max_retries: 0
+ 
+ # Initial delay in seconds before the first retry attempt (exponential backoff)
+ # Can be overridden by PROCESSOR_INITIAL_RETRY_DELAY_SECONDS environment variable
+ initial_retry_delay_seconds: 60
--- a/config/application.yml
+++ b/config/application.yml
@ -38,16 +38,16 @@ llm:
  # Settings for Ollama
  ollama:
    # Can be overridden by OLLAMA_BASE_URL
-    # base_url: "http://192.168.0.122:11434"
-    base_url: "https://api-amer-sandbox-gbl-mdm-hub.pfizer.com/ollama"
+    base_url: "http://ollama-jira:11434"
+    # base_url: "https://api-amer-sandbox-gbl-mdm-hub.pfizer.com/ollama"


    # Can be overridden by OLLAMA_MODEL
-    model: "phi4-mini:latest"
+    # model: "phi4-mini:latest"
    # model: "qwen3:1.7b"
    # model: "smollm:360m"
    # model: "qwen3:0.6b"
-    # model: "qwen3:8b"
+    model: "qwen3:4b"
 # Langfuse configuration for observability and analytics
 langfuse:
  # Enable or disable Langfuse integration
@ -59,7 +59,8 @@ langfuse:
  # instead of saving them in this file
  public_key: "pk-lf-"
  secret_key: "sk-lf-"
-  host: "https://cloud.langfuse.com"
+  # host: "https://cloud.langfuse.com"
+  # host: "http://192.168.0.122:3000"

 # Processor configuration
 processor:
--- a/data/DCR_mappings_ALL.md
+++ b/data/DCR_mappings_ALL.md
@ -0,0 +1,89 @@
+The `/dcr` API serves as a canonical interface for initiating data change requests. The JSON payload sent to this endpoint contains a standardized representation of the desired changes. The MDM HUB is then responsible for validating this request and mapping its attributes to the distinct models of the target systems.
+
+***
+
+### **`/dcr` API to Target System Attribute Mapping**
+
+The following sections break down the mapping from the attributes in the `/dcr` request payload to their corresponding fields in OneKey and Veeva.
+
+---
+
+#### **1. Request-Level Attributes**
+
+These attributes are defined at the top level of each object within the `DCRRequests` array in the JSON payload.
+
+| HUB /dcr Attribute | OneKey Mapping | Veeva OpenData (VOD) Mapping | Notes / Description |
+| :--- | :--- | :--- | :--- |
+| **`extDCRRequestId`** | Used to populate the `DCRID` in the Reltio DCR tracking entity. OneKey's `validation.clientRequestId` is typically a HUB-generated internal ID, but `extDCRRequestId` is the primary key for client-side tracking. | **`dcr_key`** (in all CSV files: `change_request.csv`, `change_request_hco.csv`, etc.) | **This is the primary external identifier for the entire DCR.** It is crucial for clients like PforceRx to track the request's status and is used as the main correlation ID across all systems and files. |
+| **`extDCRComment`** | **`validation.requestComment`** | **`description`** (in `change_request.csv`) | A free-text field for the requester to provide context or justification for the DCR. For OneKey, it has a special function: due to API limitations, requests to **remove** an attribute are specified in this comment field (e.g., "Please remove attributes: [Address: ...]"). |
+| **`country`** | **`isoCod2`** | **`primary_country__v`** (in `change_request_hcp.csv` and `change_request_hco.csv`) and **`country__v`** (in `change_request_address.csv`) | The mandatory two-letter ISO country code. This is a critical routing attribute that determines which validator instance to use and, for Veeva, which S3/SFTP directory the DCR files are placed in. |
+| **`action`** (within HCP/HCO object) | This determines the logic. `update` and `insert` map to a `submitVR` call. An `update` requires a valid `individual.individualEid` or `workplace.workplaceEid`. `delete` is handled by updating the entity's end date in Reltio. | **`change_request_type`** (in `change_request.csv`). Mapped to **`ADD_REQUEST`** for an `insert` action, and **`CHANGE_REQUEST`** for an `update` action. | This defines the fundamental operation being performed on the entity. |
+| **`refId`** (within HCP/HCO object) | Used to query Reltio to find the OneKey crosswalk (`individual.individualEid` or `workplace.workplaceEid`) which is mandatory for update operations. | Used to query Reltio to find the Veeva crosswalk (`vid__v`), which is mandatory for update operations. The Reltio URI from the `refId` is also used to populate the **`entity_key`** field in the VOD CSVs. | This object contains the necessary identifiers (`CrosswalkTargetObjectId`, `EntityURITargetObjectId`, etc.) to locate the target entity in Reltio for an update or delete operation. |
+
+---
+
+#### **2. Healthcare Professional (HCP) Attributes**
+
+These attributes are provided within the `HCP` object of a DCR request.
+
+| HUB /dcr Attribute | OneKey Mapping | Veeva OpenData (VOD) Mapping | Notes / Description |
+| :--- | :--- | :--- | :--- |
+| **`firstName`** | `individual.firstName` | `first_name__v` | HCP's first name. |
+| **`lastName`** | `individual.lastName` | `last_name__v` | HCP's last name. Mandatory for creating a new HCP in OneKey. |
+| **`middleName`** | `individual.middleName` | `middle_name__v` | HCP's middle name. |
+| **`prefix`** | `individual.prefixNameCode` | `prefix__v` | Name prefix (e.g., Mr., Dr.). Requires a lookup from the canonical code (`HCPPrefix`) to the target system's specific code. |
+| **`title`** | `individual.titleCode` | `professional_title__v` | Professional title. Requires a lookup from the canonical code (`HCPTitle` or `HCPProfessionalTitle`) to the target system's specific code. |
+| **`gender`** | `individual.genderCode` | `gender__v` | HCP's gender. Requires a lookup to map the canonical value (e.g., M, F) to the target system's code. |
+| **`subTypeCode`** | `individual.typeCode` | `hcp_type__v` | The professional subtype of the HCP (e.g., Physician, Nurse). Requires a lookup from the canonical code (`HCPSubTypeCode` or `HCPType`). |
+| **`specialties`** (List) | `individual.speciality1`, `individual.speciality2`, `individual.speciality3` | `specialty_1__v` through `specialty_10__v` | The list of specialties is **flattened**. OneKey accepts up to 3 ranked specialties. Veeva accepts up to 10. A lookup is required to map the canonical `HCPSpecialty` code to the target system's value. |
+| **`emails`** (List) | `individual.email` | `email_1__v`, `email_2__v` | The list of emails is flattened. OneKey typically takes the highest-ranked email. Veeva takes the top two. |
+| **`phones`** (List) | `individual.mobilePhone` | `phone_1__v` to `phone_3__v` (for office), `fax_1__v` to `fax_2__v` (for fax) | The list is filtered by type and ranked. OneKey maps the highest-ranked to `mobilePhone`. Veeva separates numbers into distinct `phone` and `fax` columns based on the Reltio phone type. |
+
+---
+
+#### **3. Healthcare Organization (HCO) Attributes**
+
+These attributes are provided within the `HCO` object of a DCR request.
+
+| HUB /dcr Attribute | OneKey Mapping | Veeva OpenData (VOD) Mapping | Notes / Description |
+| :--- | :--- | :--- | :--- |
+| **`name`** | `workplace.usualName` / `workplace.officialName` | `corporate_name__v` | The primary, official name of the organization. |
+| **`otherNames`** (List) | `workplace.usualName2` | `alternate_name_1__v` | The list of alternative names is flattened. Both systems typically take the first or highest-ranked alternative name. |
+| **`subTypeCode`** | `workplace.typeCode` | `major_class_of_trade__v` | The HCO's subtype, often representing facility type. Requires a lookup from the canonical code (`COTFacilityType`). |
+| **`typeCode`** | Not Mapped | `hco_type__v` | Maps to the HCO Type. Requires a lookup (`HCOType`). The OneKey mapping document indicates this is not used for their system. |
+| **`websiteURL`** | `workplace.website` | `URL_1__v` | The official website of the organization. |
+| **`specialties`** (List) | `workplace.speciality1` to `speciality3` | `specialty_1__v` to `specialty_10__v` | Similar to HCPs, the list of specialties is flattened and ranked. A lookup from the canonical `COTSpecialty` code is required. |
+| **`emails`** (List) | `workplace.email` | `email_1__v`, `email_2__v` | List of emails is flattened, with the highest-ranked ones being used. |
+| **`phones`** (List) | `workplace.telephone`, `workplace.fax` | `phone_1__v` to `phone_3__v`, `fax_1__v` to `fax_2__v` | Similar to HCPs, the list is filtered by type (`TEL.OFFICE` vs. `TEL.FAX`) and ranked before mapping. |
+
+---
+
+#### **4. Nested Object Mapping: Addresses**
+
+Address information is provided as a list of objects within the `HCP` or `HCO` DCR payload.
+
+| HUB /dcr `addresses` Attribute | OneKey Mapping | Veeva OpenData (VOD) Mapping | Notes / Description |
+| :--- | :--- | :--- | :--- |
+| **(Address Object)** | Mapped to a single `address` complex object in the JSON request. Only the primary address is sent. | Each address object is mapped to a **separate row** in the **`change_request_address.csv`** file. | OneKey's API takes a single address per DCR, while Veeva's file-based approach can handle multiple address changes in one DCR. |
+| **`refId`** | Used to match and update an existing address. | **`address_key`** | This is the `PfizerAddressID`, a unique identifier for the address record in Reltio. |
+| **`addressLine1`** | `address.longLabel` or `address.addressLine1` | `address_line_1__v` | The first line of the street address. |
+| **`addressLine2`** | `address.longLabel2` or `address.addressLine2` | `address_line_2__v` | The second line of the street address. |
+| **`city`** | `address.city` | `locality__v` | The city name. |
+| **`stateProvince`** | `address.countyCode` | `administrative_area__v` | The state or province. Requires a lookup from the canonical value to the target system's code. |
+| **`zip`** | `address.longPostalCode` or `address.Zip5` | `postal_code__v` | The postal or ZIP code. |
+| **`addressType`** | Not explicitly mapped in `submitVR` request, but used in logic. | `address_type__v` | The type of address (e.g., Office, Home). Requires a lookup (`AddressType`). |
+
+---
+
+#### **5. Nested Object Mapping: Affiliations**
+
+Affiliations are provided as a list of objects (`contactAffiliations` for HCP, `otherHCOAffiliations` for HCO) in the DCR payload.
+
+| HUB /dcr `affiliations` Attribute | OneKey Mapping | Veeva OpenData (VOD) Mapping | Notes / Description |
+| :--- | :--- | :--- | :--- |
+| **(Affiliation Object)** | The `refId` of the target HCO is used to find its `workplace.workplaceEid`, which is then sent as part of the HCP or parent HCO record. | Each affiliation object is mapped to a **separate row** in the **`change_request_parenthco.csv`** file. | The mapping for affiliations is fundamentally different. OneKey handles it as an attribute of the child entity, while Veeva treats it as a distinct relationship record. |
+| **Relation `refId`** | The `workplace.workplaceEid` of the target (parent) HCO is populated in the request. | **`parenthco_key`** (populated with the Reltio `relationUri`) | Veeva requires a unique key for the relationship itself. |
+| **Start/Child Object** | The main body of the request contains the child's details (e.g., the HCP). | **`child_entity_key`** | This field is populated with the `entity_key` of the child entity (e.g., the HCP). |
+| **End/Parent Object** | The `workplace.workplaceEid` of the parent HCO is sent. | **`parent_entity_key`** | This field is populated with the `entity_key` of the parent entity (e.g., the HCO). |
+| **`type`** | `activity.role` (requires lookup, e.g., `TIH.W.*`) | **`relationship_type__v`** (requires lookup from `RelationType`) | The type or role of the affiliation (e.g., Employed, Manages). |
+| **`primary`** | Not explicitly mapped. | **`is_primary_relationship__v`** | A boolean flag (`Y`/`N`) indicating if this is the primary affiliation. |
--- a/data/DCR_process_overview.md
+++ b/data/DCR_process_overview.md
@ -0,0 +1,191 @@
+### **1. Overall DCR Process Overview**
+
+*   **Purpose of DCR Process:**
+    The Data Change Request (DCR) process is designed to improve and validate the quality of existing data in source systems. It provides a formal mechanism for proposing, validating, and applying data changes.
+
+*   **General DCR Process Flow:**
+    1.  A source system creates a proposal for a data change, known as a DCR or Validation Request (VR).
+    2.  The MDM HUB routes this request to the appropriate validation channel.
+    3.  Validation is performed either by internal Data Stewards (DS) within Reltio or by external, third-party validator services like OneKey or Veeva OpenData.
+    4.  A response is sent back, which includes metadata about the DCR's status (e.g., accepted, rejected) and the actual data profile update (payload) resulting from the processed DCR.
+
+*   **High-Level Solution Architecture:**
+    The architecture involves source systems initiating DCRs through the **MDM HUB**. The HUB acts as a central router, directing requests to either **Reltio** for internal data steward review or to **Third-Party Validators** (OneKey, Veeva). The HUB is also responsible for receiving responses and facilitating the update of data in Reltio, often through ETL processes that handle payload delivery (e.g., via S3).
+
+***
+
+### **2. System-Specific DCR Implementations**
+
+Here are the details for each system involved in the DCR process.
+
+---
+#### **OneKey (OK)**
+
+*   **Role in DCR Process:** Functions as a third-party validator for DCRs. It receives validation requests, processes them, and returns the results.
+*   **Key Components:** DCR Service 2, OK DCR Service, OneKey Adapter, Publisher, Hub Store (Mongo DB), Manager (Reltio Adapter).
+*   **Actors Involved:** PforceRX, Data Stewards (in Reltio), Reltio, HUB, OneKey.
+*   **Core Process Details:**
+    *   **PforceRX-initiated:** DCRs are created via the HUB's API. The HUB integrates with OneKey's API to submit requests (`/vr/submit`) and periodically checks for status updates (`/vr/trace`).
+    *   **Reltio-initiated:** Data Stewards can suggest changes in Reltio and use the "Send to Third Party Validation" feature, which triggers a flow to submit a validation request to OneKey. Singleton entities created in Reltio can also trigger an automatic validation request to OneKey.
+*   **Integration Methods:**
+    *   **API:** Real-time integration with OneKey via REST APIs (`/vr/submit`, `/vr/trace`).
+    *   **File Transfer:** Data profile updates (payload) are delivered back to Reltio via CSV files on an S3 bucket, which are then processed by an ETL job.
+*   **DCR Types Handled:** Create, update, delete operations for HCP/HCO profiles; validation of newly created singleton entities; changes suggested by Data Stewards.
+
+---
+#### **Veeva OpenData (VOD)**
+
+*   **Role in DCR Process:** Functions as a third-party validator, primarily handling DCRs initiated by Data Stewards from within Reltio.
+*   **Key Components:** DCR Service 2, Veeva DCR Service, Veeva Adapter, GMTF (Global Master Template & Foundation) jobs.
+*   **Actors Involved:** Data Stewards (in Reltio), HUB, Veeva OpenData.
+*   **Core Process Details:**
+    1.  Data Stewards in Reltio create DCRs using the "Suggest / Send to 3rd Party Validation" functionality.
+    2.  The HUB stores these requests in a Mongo collection (`DCRRegistryVeeva`).
+    3.  A scheduled job gathers these DCRs, packages them into ZIP files containing multiple CSVs, and places them on an S3 bucket.
+    4.  Files are synchronized from S3 to Veeva's SFTP server in batches (typically every 24 hours).
+    5.  Veeva processes the files and returns response files to an inbound S3 directory, which the HUB traces to update DCR statuses.
+*   **Integration Methods:**
+    *   **File Transfer:** Asynchronous, batch-based communication via ZIP/CSV files exchanged through S3 and SFTP.
+*   **DCR Types Handled:** Primarily handles changes suggested by Data Stewards for existing profiles that need external validation from Veeva.
+
+---
+#### **IQVIA Highlander (HL) - *Decommissioned April 2025***
+
+*   **Role in DCR Process:** Acted as a wrapper to translate DCRs from a Veeva format into a format that could be loaded into Reltio for Data Steward review.
+*   **Key Components:** DCR Service (first version), IQVIA DCR Wrapper.
+*   **Actors Involved:** Veeva (on behalf of PforceRX), Reltio, HUB, IQVIA wrapper, Data Stewards.
+*   **Core Process Details:**
+    1.  Veeva uploaded DCR requests as CSV files to an FTP location.
+    2.  The HUB translated the Veeva CSV format into the IQVIA wrapper's CSV format.
+    3.  The IQVIA wrapper processed this file and created DCRs directly in Reltio.
+    4.  Data Stewards would then review, approve, or reject these DCRs within Reltio.
+*   **Integration Methods:**
+    *   **File Transfer:** Communication was entirely file-based via S3 and SFTP.
+*   **DCR Types Handled:** Aggregated 21 specific use cases into six generic types: `NEW_HCP_GENERIC`, `UPDATE_HCP_GENERIC`, `DELETE_HCP_GENERIC`, `NEW_HCO_GENERIC`, `UPDATE_HCO_GENERIC`, `DELETE_HCO_GENERIC`.
+
+***
+
+### **3. Key DCR Operations and Workflows**
+
+---
+#### **Create DCR**
+
+*   **Description:** This is the main entry point for clients like PforceRx to create DCRs. The process validates the request, routes it to the correct target system (Reltio, OneKey, or Veeva), and creates the DCR.
+*   **Triggers:** An API call to `POST /dcr`.
+*   **Detailed Steps:**
+    1.  The DCR service receives and validates the request (e.g., checks for duplicate IDs, existence of referenced objects for updates).
+    2.  It uses a decision table to determine the target system based on attributes like country, source, and operation type.
+    3.  It calls the appropriate internal method to create the DCR in the target system (Reltio, OneKey, or Veeva).
+    4.  A corresponding DCR tracking entity is created in Reltio, and the state is saved in the Mongo DCR Registry.
+    5.  For Reltio-targeted DCRs, a workflow is initiated for Data Steward review.
+    6.  Pre-close logic may be applied to auto-accept or auto-reject the DCR based on the country.
+*   **Decision Logic/Rules:** A configurable decision table routes DCRs based on `userName`, `sourceName`, `country`, `operationType`, `affectedAttributes`, and `affectedObjects`.
+
+---
+#### **Submit Validation Request**
+
+*   **Description:** This process submits validation requests for newly created "singleton" entities in Reltio to the OneKey service.
+*   **Triggers:** Reltio events (e.g., `HCP_CREATED`, `HCO_CREATED`) are aggregated in a time window (e.g., 4 hours).
+*   **Detailed Steps:**
+    1.  After an event aggregation window closes, the process performs several checks (e.g., entity is active, no existing OneKey crosswalk, no potential matches found via Reltio's `getMatches` API).
+    2.  If all checks pass, the entity data is mapped to a OneKey `submitVR` request.
+    3.  The request is sent to OneKey via `POST /vr/submit`.
+    4.  A DCR entity is created in Reltio to track the status, and the request is logged in Mongo.
+
+---
+#### **Trace Validation Request**
+
+*   **Description:** This scheduled process checks the status of pending validation requests that have been sent to an external validator like OneKey or Veeva.
+*   **Triggers:** A timed scheduler (cron job) that runs every N hours.
+*   **Detailed Steps (OneKey Example):**
+    1.  The process queries the Mongo DCR cache for requests with a `SENT` status.
+    2.  For each request, it calls the OneKey `POST /vr/trace` API.
+    3.  It evaluates the `processStatus` and `responseStatus` from the OneKey response.
+    4.  If the request is resolved (`VAS_FOUND`, `VAS_NOT_FOUND`, etc.), the DCR status in Reltio and Mongo is updated to `ACCEPTED` or `REJECTED`.
+    5.  If the response indicates a match was found but an OK crosswalk doesn't yet exist in Reltio, a new workflow is triggered for Data Steward manual review (`DS_ACTION_REQUIRED`).
+
+---
+#### **Data Steward Response**
+
+*   **Description:** This process handles the final outcome of a DCR that was reviewed internally by a Data Steward in Reltio.
+*   **Triggers:** Reltio change request events (`CHANGE_REQUEST_CHANGED`, `CHANGE_REQUEST_REMOVED`) that **do not** have the `ThirdPartyValidation` flag.
+*   **Detailed Steps:**
+    1.  The process consumes the event from Reltio.
+    2.  It checks the `state` of the change request.
+    3.  If the state is `APPLIED` or `REJECTED`, the corresponding DCR entity in Reltio and the record in Mongo are updated to a final status of `ACCEPTED` or `REJECTED`.
+
+---
+#### **Data Steward OK Validation Request**
+
+*   **Description:** This process handles DCRs created by a Data Steward in Reltio using the "Suggest" and "Send to Third Party Validation" features, routing them to an external validator like OneKey.
+*   **Triggers:** Reltio change request events that **do** have the `ThirdPartyValidation` flag set to `true`.
+*   **Detailed Steps:**
+    1.  The HUB retrieves the "preview" state of the entity from Reltio to see the suggested changes.
+    2.  It compares the current entity with the preview to calculate the delta.
+    3.  It maps these changes to a OneKey `submitVR` request. Attribute removals are sent as a comment due to API limitations.
+    4.  The request is sent to OneKey.
+    5.  Upon successful submission, the original change request in Reltio is programmatically rejected (since the validation is now happening externally), and a new DCR entity is created for tracking the OneKey validation.
+
+***
+
+### **4. Data Comparison and Mapping Details**
+
+*   **OneKey Comparator:**
+    When a Data Steward suggests changes, the HUB compares the current Reltio entity with the "preview" state to send to OneKey.
+    *   **Simple Attributes** (e.g., `FirstName`): Values are compared for equality. The suggested value is taken if different.
+    *   **Complex Attributes** (e.g., `Addresses`, `Specialties`): Nested attributes are matched using their Reltio URI. New nested objects are added, and changes to existing ones are applied.
+    *   **Mandatory Fields:** For HCP, `LastName` and `Country` are mandatory. For HCO, `Country` and `Addresses` are mandatory.
+    *   **Attribute Removal:** Due to API limitations, removing an attribute is not done directly but by generating a comment in the request, e.g., "Please remove attributes: [Address: ...]".
+
+*   **Veeva Mapping:**
+    The process of mapping Reltio canonical codes to Veeva's source-specific codes is multi-layered.
+    1.  **Veeva Defaults:** The system first checks custom CSV mapping files stored in configuration (`mdm-veeva-dcr-service/defaults`). These files define direct mappings for a specific country and canonical code (e.g., `IN;SP.PD;PD`).
+    2.  **RDM Lookups:** If no default is found, it queries RDM (via a Mongo `LookupValues` collection) for the canonical code and looks for a `sourceMapping` where the source is "VOD".
+    3.  **Veeva Fallback:** If no mapping is found, it consults fallback CSV files (`mdm-veeva-dcr-service/fallback`) for certain attributes (e.g., `hco-specialty.csv`). A regular expression is often used to extract the correct code. If all else fails, a question mark (`?`) is used as the default fallback.
+
+***
+
+### **5. Status Management and Error Handling**
+
+*   **DCR Statuses:**
+    The system uses a combination of statuses to track the DCR lifecycle.
+
+| RequestStatus | DCRStatus | Internal Cache Status | Description |
+| :--- | :--- | :--- | :--- |
+| REQUEST\_ACCEPTED | CREATED | SENT\_TO\_OK | DCR sent to OneKey, pending DS review. |
+| REQUEST\_ACCEPTED | CREATED | SENT\_TO\_VEEVA | DCR sent to Veeva, pending DS review. |
+| REQUEST\_ACCEPTED | CREATED | DS\_ACTION\_REQUIRED | DCR pending internal DS validation in Reltio. |
+| REQUEST\_ACCEPTED | ACCEPTED | ACCEPTED | DS accepted the DCR; changes were applied. |
+| REQUEST\_ACCEPTED | ACCEPTED | PRE\_ACCEPTED | Pre-close logic automatically accepted the DCR. |
+| REQUEST\_REJECTED | REJECTED | REJECTED | DS rejected the DCR. |
+| REQUEST\_REJECTED | REJECTED | PRE\_REJECTED | Pre-close logic automatically rejected the DCR. |
+| REQUEST\_FAILED | - | FAILED | DCR failed due to a validation or system error. |
+
+*   **Error Codes:**
+
+| Error Code | Description | HTTP Code |
+| :--- | :--- | :--- |
+| `DUPLICATE_REQUEST` | The `extDCRRequestId` has already been registered. | 403 |
+| `NO_CHANGES_DETECTED` | The request contained no changes compared to the existing entity. | 400 |
+| `VALIDATION_ERROR` | A referenced object (HCP/HCO) or attribute could not be found. | 404 / 400 |
+
+*   **DCR State Changes:**
+    A DCR begins in an `OPEN` state. It is then sent to a target system, moving to states like `SENT_TO_OK`, `SENT_TO_VEEVA`, or `DS_ACTION_REQUIRED`. Pre-close logic can immediately move it to `PRE_ACCEPTED` or `PRE_REJECTED`. Following data steward review (either internal or external), the DCR reaches a terminal state of `ACCEPTED` or `REJECTED`. If an error occurs, it moves to `FAILED`.
+
+***
+
+### **6. Technical Artifacts and Infrastructure**
+
+*   **Mongo Collections:**
+    *   **`DCRRegistryONEKEY` / `DCRRegistryVeeva`:** System-specific collections to store and track the state of DCRs sent to OneKey and Veeva, respectively. They hold the mapped request data and trace the response.
+    *   **`DCRRequest` / `DCRRegistry`:** General-purpose collections for tracking DCRs, their metadata, and overall status within the HUB.
+    *   **`DCRRegistryVeeva`:** Specifically stores Veeva-bound DCRs, including the raw CSV line content, before they are submitted in a batch.
+
+*   **File Specifications:**
+    *   **Veeva Integration:** Uses `ZIP` files containing multiple `CSV` files (`change_request.csv`, `change_request_hcp.csv`, `change_request_address.csv`, etc.). Response files follow a similar pattern and are named with the convention `<country>_DCR_Response_<Date>.zip`.
+    *   **Highlander Integration:** Used `CSV` files for requests.
+
+*   **Event Models:**
+    The system uses internal Kafka events to communicate DCR status changes between components.
+    *   **`OneKeyDCREvent`:** Published by the trace process after receiving a response from OneKey. It contains the DCR ID and the `OneKeyChangeRequest` details (`vrStatus`, `vrStatusDetail`, comments, validated IDs).
+    *   **`VeevaDCREvent`:** Published by the Veeva trace process. It contains the DCR ID and `VeevaChangeRequestDetails` (`vrStatus`, `vrStatusDetail`, comments, new Veeva IDs).
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,14 +1,14 @@
 name: jira-webhook-stack
 services:
  ollama-jira:
-    image: artifactory.pfizer.com/mdmhub-docker-dev/mdmtools/ollama/ollama-preloaded:0.0.1
+    image: artifactory.pfizer.com/mdmhub-docker-dev/mdmtools/ollama/ollama-preloaded:0.0.2
    ports:
      - "11434:11434"
    restart: unless-stopped

  # Service for your FastAPI application
  jira-webhook-llm:
-    image: artifactory.pfizer.com/mdmhub-docker-dev/mdmtools/ollama/jira-webhook-llm:0.2.0
+    image: artifactory.pfizer.com/mdmhub-docker-dev/mdmtools/ollama/jira-webhook-llm:0.2.5
    ports:
      - "8000:8000"
    environment:
@ -17,10 +17,12 @@ services:
      
      # Point to the Ollama service within the Docker Compose network
      # 'ollama' is the service name, which acts as a hostname within the network
-      OLLAMA_BASE_URL: "https://api-amer-sandbox-gbl-mdm-hub.pfizer.com/ollama"
+      # OLLAMA_BASE_URL: "https://api-amer-sandbox-gbl-mdm-hub.pfizer.com/ollama"
+      OLLAMA_BASE_URL: "http://ollama-jira:11434"

      # Specify the model to use
-      OLLAMA_MODEL: phi4-mini:latest
+      # OLLAMA_MODEL: phi4-mini:latest
+      OLLAMA_MODEL: qwen3:4b

    # Ensure the Ollama service starts and is healthy before starting the app
    depends_on:
--- a/llm/chains.py
+++ b/llm/chains.py
@ -2,6 +2,7 @@ import json
 import sys
 from typing import Union, Any # Import Any
 from pydantic import SecretStr # Re-import SecretStr
+import re # Import re for regex operations

 from langchain_core.prompts import (
    ChatPromptTemplate,
@ -60,7 +61,7 @@ elif settings.llm.mode == 'ollama':
        import os
        if os.getenv("IS_TEST_ENV") != "true":
            logger.debug("Testing Ollama connection...")
-            llm.invoke("test")  # Simple test request
+            llm.invoke("test /no_think ")  # Simple test request
            logger.info("Ollama connection established successfully")
        else:
            logger.info("Skipping Ollama connection test in test environment.")
@ -99,7 +100,7 @@ elif settings.llm.mode == 'gemini': # New: Add Gemini initialization
        import os
        if os.getenv("IS_TEST_ENV") != "true":
            logger.debug("Testing Gemini connection...")
-            llm.invoke("test") # Simple test request
+            llm.invoke("test /no_think ") # Simple test request
            logger.info("Gemini connection established successfully")
        else:
            logger.info("Skipping Gemini connection test in test environment.")
@ -133,7 +134,7 @@ llm_runnable: Runnable = llm # type: ignore
 parser = JsonOutputParser()

 # Load prompt template from file
-def load_prompt_template(version="v1.2.0"):
+def load_prompt_template(version="v1.3.0"):
    try:
        with open(f"llm/jira_analysis_{version}.txt", "r") as f:
            template_content = f.read()
@ -158,13 +159,35 @@ def load_prompt_template(version="v1.2.0"):
 # Fallback prompt template
 FALLBACK_PROMPT = ChatPromptTemplate.from_messages([
    SystemMessagePromptTemplate.from_template(
-        "Analyze Jira tickets and output JSON with hasMultipleEscalations, customerSentiment"
+        "Analyze Jira tickets and output JSON with hasMultipleEscalations"
    ),
    HumanMessagePromptTemplate.from_template(
        "Issue Key: {issueKey}\nSummary: {summary}"
    )
 ])

+# Helper function to extract JSON from a string that might contain other text
+def extract_json(text: str) -> str:
+    """
+    Extracts the first complete JSON object from a string.
+    Assumes the JSON object is enclosed in curly braces {}.
+    """
+    # Find the first occurrence of '{'
+    start_index = text.find('{')
+    if start_index == -1:
+        logger.warning(f"No opening curly brace found in LLM response: {text}")
+        return text # Return original text if no JSON start is found
+
+    # Find the last occurrence of '}'
+    end_index = text.rfind('}')
+    if end_index == -1 or end_index < start_index:
+        logger.warning(f"No closing curly brace found or invalid JSON structure in LLM response: {text}")
+        return text # Return original text if no JSON end is found or it's before the start
+
+    json_string = text[start_index : end_index + 1]
+    logger.debug(f"Extracted JSON string: {json_string}")
+    return json_string
+
 # Create chain with fallback mechanism
 def create_analysis_chain():
    try:
@ -183,6 +206,7 @@ def create_analysis_chain():
            }
            | prompt_template
            | llm_runnable # Use the explicitly typed runnable
+            | extract_json # Add the new extraction step
            | parser
        )
        
--- a/llm/jira_analysis_v1.2.0.txt
+++ b/llm/jira_analysis_v1.2.0.txt
@ -25,12 +25,7 @@ Your output MUST be a single JSON object and nothing else.
    *   `"Authentication"`
    *   `"Other"`

-3.  **`customerSentiment` (string)**
-    *   Analyze the user's tone.
-    *   `"frustrated"`: User mentions blockers, deadlines, or uses urgent language ("ASAP", "urgent", "blocked").
-    *   `"neutral"`: Default, non-emotional tone.
-
-4.  **`isEscalated` (boolean)**
+3.  **`isEscalated` (boolean)**
    *   Set to `true` if the user explicitly states they are escalating, mentions previous unanswered requests, or if the tone is highly frustrated and urgent.
    *   Set to `false` otherwise.

@ -49,7 +44,6 @@ Your output MUST be a single JSON object and nothing else.
 {{
  "issueCategory": "technical_issue",
  "area": "Application Service",
-  "customerSentiment": "frustrated",
  "isEscalated": false,
  "oneSentenceSummary": "A DCR for PforceRx was rejected by OneKey, which is blocking a weekly report."
 }}
--- a/llm/jira_analysis_v1.3.0.txt
+++ b/llm/jira_analysis_v1.3.0.txt
@ -0,0 +1,92 @@
+SYSTEM:
+You are an expert AI assistant that analyzes Jira tickets and outputs a concise summary in a valid JSON format.
+Your output MUST be a single JSON object and nothing else.
+
+## JSON Output Schema
+{format_instructions}
+
+## Field-by-Field Instructions
+
+1.  **`issueCategory` (string)**
+    *   Classify the core problem. Choose ONE: "technical_issue", "data_request", "access_problem", "general_question", "other".
+
+2.  **`area` (string)**
+    *   Classify the technical domain. Choose the BEST fit from the following options:
+    *   `"Direct Channel"`
+    *   `"Streaming Channel"`
+    *   `"Java Batch Channel"`
+    *   `"ETL Batch Channel"`
+    *   `"DCR Service"`
+    *   `"API Gateway"`
+    *   `"Callback Service"`
+    *   `"Publisher"`
+    *   `"Reconciliation"`
+    *   `"Snowflake"`
+    *   `"Authentication"`
+    *   `"Other"`
+
+3.  **`urgencyScore` (float)**
+    *   Estimate the client's urgency and frustration level as a float between 0.0 and 1.0.
+    *   **0.0 - 0.2 (Low Urgency)**: Calm, routine inquiries. No explicit or implicit signs of frustration or urgency.
+    *   **0.3 - 0.5 (Moderate Urgency)**: Client is seeking information, but without strong emotional indicators. May mention minor inconveniences or follow-ups without demanding immediate action.
+    *   **0.6 - 0.8 (High Urgency)**: Client expresses clear frustration, uses words like "urgent," "high priority," "ASAP," "blocked," "critical," or implies delays are causing significant issues. May mention previous unanswered requests or multiple attempts to get attention.
+    *   **0.9 - 1.0 (Critical Urgency)**: Client is highly frustrated, demanding immediate resolution, or explicitly stating an escalation. This indicates a severe impact or a breakdown in communication.
+    *   Infer urgency from keywords, tone, repeated inquiries, and the impact described (e.g., "blocking our weekly report").
+
+
+5. ### `description` (string)
+- Provide a concise summary of the **current status of the problem**, specifically detailing any **actions or verifications that have already been completed or confirmed.** Focus on what is known or has been done up to this point regarding the described issue.
+
+6. ### `actions` (string)
+- Generate a multi-line, formatted text block that summarizes all actionable items, open topics, considerations, and next steps. Each point MUST start with a bullet (`- `) and be on a new line.
+- For each point, use the following structure based on its `type`:
+    - **Action Item:** `- [STATUS] [Assignee Name (Team)] Detailed description of the action.`
+        - **STATUS:** Use `DONE`, `IN PROGRESS`, `OPEN`.
+        - **Assignee Name:** The name of the person responsible (e.g., "Grzegorz", "Ireneusz").
+        - **Team:** The team responsible: `MDM Team`, `MDM HUB Team`, `External Team`, `Other Team`. If unknown, use `Unknown Team`.
+    - **Open Topic:** `- Open Topic: Detailed description of the unresolved issue/question.`
+    - **Consideration:** `- Consideration: Detailed description of a factor or implication to be aware of.`
+    - **Next Step:** `- Next Step: [Assignee Name (Team)] Detailed description of the next action to take.`
+- Ensure the description for each point is clear and verbose enough.
+
+7. ### `timeToResolutionDays` (number)
+- Calculate the total time in **days** from the **initial request or first mention of the problem** in the ticket to the point where the **problem was confirmed to be resolved or a fix was delivered/implemented**.
+- **Steps for calculation:**
+    1.  Identify the earliest date of communication (email or comment) describing the problem.
+    2.  Identify the latest date of communication (email or comment) explicitly stating the problem's resolution, fix delivery, or successful verification.
+    3.  Calculate the difference in whole days between these two dates.
+- If either date cannot be identified, set this value to `null`.
+
+
+## Example (Using your provided User section for calculation)
+
+### Input:
+- Summary: "RE: Failure in Global MDM Ex-US interface"
+- Description: "++MDM and MDM Hub team Hi MDM and MDM HUB team, Can you please look into this and revert as soon as possible ? Regards Saraswati *From:* Choudhary, Anita *Sent:* Monday, June 16, 2025 5:27 PM *To:* Moses, Donie *Cc:* Subramanya, Suraj K *Subject:* RE: Failure in Global MDM Ex-US interface Hi Donie, There has not been any activity from support end which I have understood from the team which have impacted this. Upon comparing the P_HCP structure in PROD and QA it looks like there are 116 columns in QA and 128 in PROD. We could observe 13 fields are less in QA.PFB: [@Das, Deepanjan] – Could you please check at your end # If any ongoing activity has impacted this # Was there any change in DDL of P_HCP in QA # When was the last DDL changed and by whom if possible. Thanks & Regards, Anita *From:* Moses, Donie <[Donie.Moses@pfizer.com]> *Sent:* Monday, June 16, 2025 3:58 PM *To:* Sahu, Saraswati <[Saraswati.Sahu@pfizer.com]> *Cc:* Subramanya, Suraj K <[Suraj.KSubramanya@pfizer.com]> *Subject:* RE: Failure in Global MDM Ex-US interface [@Choudhary, Anita] Could you get this checked please ? Thanks, Donie * *Donie Moses** * *Manager – Data Integration Support** * *Pfizer Digital Global Support and Operations** Office: +44-7587 442339 Mobile: +44-7483 255494 [donie.moses@pfizer.com] Address: Pfizer Ltd., Dorking Road, Walton Oaks, Tadworth, Surrey. KT20 7NS *From:* Sahu, Saraswati <[Saraswati.Sahu@pfizer.com]> *Sent:* Monday, 16 June 2025 10:03 *To:* B, Pavithra <[Pavithra.B@pfizer.com]> *Cc:* Choudhary, Anita <[Anita.Choudhary@pfizer.com]> *Subject:* Failure in Global MDM Ex-US interface Hi Pavithra, Yasaswini, While testing the Global MDM Ex-US Interface in lower environment ,we have encountered one failure because IS_SPEKAER field has been removed from P_HCP table. This field is available in Production environment. Are you aware of this change ? Can you check with MDM team why this field has been removed in lower environment ? Regards Saraswati"
+- Status: "Resolved"
+- Latest Comment (excerpt for context): "==COMMENT_NOTE_13 COMMENT_NOTE_13_AUTHOR: Bachanowicz, Mieczysław (Irek) COMMENT_NOTE_13_DATE: 2025-06-25T10:23:16.8-0400 COMMENT_NOTE_13_BODY: Hi [ @Subramanya, Suraj K] First, I apologize for the lengthy delay—this situation was not straightforward, and standard SOPs did not address the problem. The issue with the missing column has been resolved. Please verify on your side to ensure everything is working correctly. The root cause was the recent deletion of US data from EMEA cluster. IS_SPEAKER column was populated only with US-related records. Our procedures for updating the data model determined that if there were no data remaining in the column, the column itself was unnecessary—hence, it was removed. At this point, we have implemented a temporary fix. Tomorrow, we will work on developing a permanent solution to prevent similar issues in the future. Regards, Mieczysław (Irek) Bachanowicz *MDM HUB team* ---- *Od:* Subramanya, Suraj K *Wysłane:* środa, 25 czerwca 2025 14:16 *Do:* Das, Deepanjan *DW:* Vedula, Anvesh C. *Temat:* RE: Failure in Global MDM Ex-US interface Hi Team, Any update on the issue ? Our testing is put on hold due to this issue. Can we please look into this on priority ? Regards, Suraj ===COMMENT_NOTE_14 COMMENT_NOTE_14_AUTHOR: Suraj.KSubramanya@pfizer.com COMMENT_NOTE_14_DATE: 2025-06-26T00:52:20.7-0400 COMMENT_NOTE_14_BODY: Thank you, Irek and team. We can see the column now. Regards, Suraj ---- ="
+
+### Output:
+```json
+{{
+  "urgencyScore": 0.7,
+  "issueCategory": "technical_issue",
+  "area": "Streaming Channel",
+  "description": "The Global MDM Ex-US Interface failed in a lower environment due to the removal of the IS_SPEAKER field from the P_HCP table, which is present in Production. MDM and MDM HUB teams were asked to investigate immediately. Initial internal comparison shows 13 fewer columns in QA vs PROD for P_HCP structure. This includes missing schema details and potential DDL changes. Manual refresh process was initiated and completed, confirming column availability on QA/STG. The root cause was identified as the recent deletion of US data from the EMEA cluster.",
+  "actions": "- [DONE] Ireneusz (MDM Team) Confirmed DCR ID in internal system for correctness. (Note: This action is from a previous example and might not fit the current ticket. The LLM should generate actions relevant to the provided input.)\\n- [DONE] Karol (Unknown Team) Manually reran refresh process.\\n- [DONE] Grzegorz (MDM HUB Team) Confirmed columns are available on QA/STG in P_HCP.\\n- [DONE] Mieczysław Bachanowicz (MDM HUB Team) Confirmed issue with missing column has been resolved.\\n- [OPEN] Mieczysław Bachanowicz (MDM HUB Team) Develop a permanent solution to prevent similar issues in the future.\\n- Open Topic: Confirm if the IS_SPEAKER field is now permanently available in QA/STG and if it will persist.\\n- Next Step: Suraj (External Team) Verify the fix on their side to ensure everything is working correctly for testing.",
+  "timeToResolutionDays": 9
+}}
+```
+
+
+USER:
+Analyze the following Jira ticket:
+
+Issue Key: {issueKey}
+Summary: {summary}
+Description: {description}
+Status: {status}
+Existing Labels: {labels}
+Assignee: {assignee}
+Last Updated: {updated}
+Latest Comment (if applicable): {comment}
--- a/llm/models.py
+++ b/llm/models.py
@ -9,11 +9,6 @@ class LLMResponse(BaseModel):
    status: str
    message: str

-class CustomerSentiment(str, Enum):
-    NEUTRAL = "neutral"
-    FRUSTRATED = "frustrated"
-    CALM = "calm"
-
 class IssueCategory(str, Enum):
    TECHNICAL_ISSUE = "technical_issue"
    DATA_REQUEST = "data_request"
@ -61,9 +56,10 @@ class AnalysisFlags(BaseModel):
    
    issueCategory: IssueCategory = Field(..., description="The primary category of the Jira ticket.")
    area: Area = Field(..., description="The technical area of the MDM HUB related to the issue.")
-    customerSentiment: Optional[CustomerSentiment] = Field(..., description="Overall customer sentiment (e.g., 'neutral', 'frustrated', 'calm').")
-    isEscalated: bool = Field(..., description="Is there evidence of multiple escalation attempts?")
-    oneSentenceSummary: str = Field(..., description="A single paragraph in concise English that summarizes the discussion.")
+    urgencyScore: float = Field(..., ge=0.0, le=1.0, description="A float between 0.0 and 1.0 indicating the estimated urgency and frustration of the client. Higher values indicate more urgency/frustration, inferred from words like 'urgent', 'high priority', 'ASAP', 'blocked', or mentions of previous unanswered requests.")
+    description: str = Field(..., description="A single paragraph in concise English that summarizes the discussion.")
+    actions: str = Field(..., description="A summary of the actions taken or recommended.")
+    timeToResolutionDays: int = Field(..., description="The estimated time to resolution in days.")


 class JiraAnalysisResponse(BaseModel):
--- a/main.py
+++ b/main.py
@ -150,7 +150,7 @@ async def lifespan(app: FastAPI):
    finally:
        # Ensure all tasks are done before cancelling the processing loop
        logger.info("Waiting for pending queue tasks to complete...")
-        requests_queue.join()
+        # requests_queue.join()
        task.cancel()
        await task # Await the task to ensure it's fully cancelled and cleaned up
        logger.info("Processing loop terminated")
--- a/shared_store.py
+++ b/shared_store.py
@ -80,6 +80,17 @@ class RequestQueue:
            completed_requests.sort(key=lambda req: req.completed_at if req.completed_at else datetime.min, reverse=True)
            return completed_requests[0] if completed_requests else None

+    def are_there_any_open_requests_for_issue_key(self, issue_key: str) -> bool:
+        """
+        Checks if there are any pending or processing requests for a given issue key.
+        """
+        with self._processing_lock:
+            for req in self._requests:
+                if req.payload.get("issueKey") == issue_key and \
+                   (req.status == RequestStatus.PENDING or req.status == RequestStatus.PROCESSING):
+                    return True
+            return False
+
    def delete_request_by_id(self, request_id: int) -> bool:
        """Deletes a specific request by its ID."""
        with self._processing_lock: