Merge branch 'prod' into gis-8825

Author: nazargesyk

uncoder-core/app/routers/meta_info.py

@@ -0,0 +1,88 @@
+from dataclasses import asdict
+
+from fastapi import APIRouter, Body, HTTPException
+
+from app.models.meta_info import (
+    MetaInfo,
+    MetaInfoResponse,
+    MitreInfoContainer,
+    MitreTacticContainer,
+    MitreTechniqueContainer,
+    ParsedLogSources,
+    RawMetaInfo,
+)
+from app.translator.core.exceptions.core import UnsupportedPlatform
+from app.translator.translator import app_translator
+
+meta_info_router = APIRouter()
+
+
+@meta_info_router.post("/get_meta_info/", tags=["meta_info"], description="Get Rule MetaInfo")
+@meta_info_router.post("/get_meta_info/", include_in_schema=False)
+def get_meta_info_data(
+    source_platform_id: str = Body(..., embed=True), text: str = Body(..., embed=True)
+) -> MetaInfoResponse:
+    try:
+        logsources, raw_query_container = app_translator.parse_meta_info(text=text, source=source_platform_id)
+    except UnsupportedPlatform as exc:
+        raise HTTPException(status_code=400, detail="Unsuported platform") from exc
+    except Exception as exc:
+        raise HTTPException(status_code=400, detail="Unexpected error.") from exc
+    if not raw_query_container:
+        raise HTTPException(status_code=400, detail="Can't parse metadata")
+    most_frequent_product = max(logsources.get("product"), key=logsources.get("product").get, default=None)
+    most_frequent_service = max(logsources.get("service"), key=logsources.get("service").get, default=None)
+    most_frequent_category = max(logsources.get("category"), key=logsources.get("category").get, default=None)
+
+    logsources.get("product", {}).pop(most_frequent_product, None)
+    logsources.get("service", {}).pop(most_frequent_service, None)
+    logsources.get("category", {}).pop(most_frequent_category, None)
+
+    parsed_logsources = ParsedLogSources(
+        most_frequent_product=most_frequent_product,
+        most_frequent_service=most_frequent_service,
+        most_frequent_category=most_frequent_category,
+        least_frequent_products=list(logsources.get("product", {}).keys()),
+        least_frequent_services=list(logsources.get("service", {}).keys()),
+        least_frequent_categories=list(logsources.get("category", {}).keys()),
+    )
+    return MetaInfoResponse(
+        query=raw_query_container.query,
+        language=raw_query_container.language,
+        meta_info=MetaInfo(
+            id_=raw_query_container.meta_info.id,
+            title=raw_query_container.meta_info.title,
+            description=raw_query_container.meta_info.description,
+            author=raw_query_container.meta_info.author,
+            date=raw_query_container.meta_info.date,
+            false_positives=raw_query_container.meta_info.false_positives,
+            license_=raw_query_container.meta_info.license,
+            mitre_attack=MitreInfoContainer(
+                tactics=[
+                    MitreTacticContainer(**asdict(tactic_container))
+                    for tactic_container in raw_query_container.meta_info.mitre_attack.tactics
+                ],
+                techniques=[
+                    MitreTechniqueContainer(**asdict(tactic_container))
+                    for tactic_container in raw_query_container.meta_info.mitre_attack.techniques
+                ],
+            ),
+            output_table_fields=raw_query_container.meta_info.output_table_fields,
+            parsed_log_sources=parsed_logsources,
+            query_fields=raw_query_container.meta_info.query_fields + raw_query_container.meta_info.function_fields,
+            query_period=raw_query_container.meta_info.query_period,
+            raw_metainfo_container=RawMetaInfo(
+                trigger_operator=raw_query_container.meta_info.raw_metainfo_container.trigger_operator,
+                trigger_threshold=raw_query_container.meta_info.raw_metainfo_container.trigger_threshold,
+                query_frequency=raw_query_container.meta_info.raw_metainfo_container.query_frequency,
+                query_period=raw_query_container.meta_info.raw_metainfo_container.query_period,
+            ),
+            raw_mitre_attack=raw_query_container.meta_info.raw_mitre_attack,
+            references=raw_query_container.meta_info.references,
+            severity=raw_query_container.meta_info.severity,
+            source_mapping_ids=raw_query_container.meta_info.source_mapping_ids,
+            status=raw_query_container.meta_info.status,
+            tags=raw_query_container.meta_info.tags,
+            timeframe=raw_query_container.meta_info.timeframe,
+        ),
+    )

uncoder-core/app/translator/core/mapping.py

@@ -22,6 +22,12 @@ class LogSourceSignature(ABC):
     def is_suitable(self, **kwargs) -> bool:
         raise NotImplementedError("Abstract method")
 
+    def is_probably_suitable(self, **kwargs) -> bool:
+        """
+        Performs check with more options, but the result is less accurate than the "is_suitable" method
+        """
+        raise NotImplementedError("Abstract method")
+
     @staticmethod
     def _check_conditions(conditions: list[Union[bool, None]]) -> bool:
         conditions = [condition for condition in conditions if condition is not None]
@@ -88,11 +94,13 @@ def __init__(
         log_source_signature: _LogSourceSignatureType = None,
         fields_mapping: Optional[FieldsMapping] = None,
         raw_log_fields: Optional[dict] = None,
+        conditions: Optional[dict] = None,
     ):
         self.source_id = source_id
         self.log_source_signature = log_source_signature
         self.fields_mapping = fields_mapping or FieldsMapping([])
         self.raw_log_fields = raw_log_fields
+        self.conditions = conditions
 
 
 class BasePlatformMappings:
@@ -123,6 +131,7 @@ def prepare_mapping(self) -> dict[str, SourceMapping]:
 
             field_mappings_dict = mapping_dict.get("field_mapping", {})
             raw_log_fields = mapping_dict.get("raw_log_fields", {})
+            conditions = mapping_dict.get("conditions", {})
             field_mappings_dict.update({field: field for field in raw_log_fields})
             fields_mapping = self.prepare_fields_mapping(field_mapping=field_mappings_dict)
             self.update_default_source_mapping(default_mapping=default_mapping, fields_mapping=fields_mapping)
@@ -131,6 +140,7 @@ def prepare_mapping(self) -> dict[str, SourceMapping]:
                 log_source_signature=log_source_signature,
                 fields_mapping=fields_mapping,
                 raw_log_fields=raw_log_fields,
+                conditions=conditions,
             )
 
         if self.skip_load_default_mappings:
@@ -170,31 +180,47 @@ def get_source_mappings_by_fields_and_log_sources(
 
         return by_log_sources_and_fields or by_fields or [self._source_mappings[DEFAULT_MAPPING_NAME]]
 
-    def get_source_mappings_by_ids(self, source_mapping_ids: list[str]) -> list[SourceMapping]:
+    def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
+        return self._source_mappings.get(source_id)
+
+    def get_source_mappings_by_ids(
+        self, source_mapping_ids: list[str], return_default: bool = True
+    ) -> list[SourceMapping]:
         source_mappings = []
         for source_mapping_id in source_mapping_ids:
+            if source_mapping_id == DEFAULT_MAPPING_NAME:
+                continue
             if source_mapping := self.get_source_mapping(source_mapping_id):
                 source_mappings.append(source_mapping)
 
-        if not source_mappings:
+        if not source_mappings and return_default:
             source_mappings = [self.get_source_mapping(DEFAULT_MAPPING_NAME)]
 
         return source_mappings
 
-    def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
-        return self._source_mappings.get(source_id)
+    def get_source_mappings_by_log_sources(self, log_sources: dict) -> Optional[list[str]]:
+        raise NotImplementedError("Abstract method")
 
     @property
     def default_mapping(self) -> SourceMapping:
         return self._source_mappings[DEFAULT_MAPPING_NAME]
 
-    def check_fields_mapping_existence(self, field_tokens: list[Field], source_mapping: SourceMapping) -> list[str]:
+    def check_fields_mapping_existence(
+        self,
+        query_field_tokens: list[Field],
+        function_field_tokens_map: dict[str, list[Field]],
+        supported_func_render_names: set[str],
+        source_mapping: SourceMapping,
+    ) -> list[str]:
         unmapped = []
-        for field in field_tokens:
-            generic_field_name = field.get_generic_field_name(source_mapping.source_id)
-            mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
-            if not mapped_field and field.source_name not in unmapped:
-                unmapped.append(field.source_name)
+
+        for field in query_field_tokens:
+            self._check_field_mapping_existence(field, source_mapping, unmapped)
+
+        for func_name, function_field_tokens in function_field_tokens_map.items():
+            if func_name in supported_func_render_names:
+                for field in function_field_tokens:
+                    self._check_field_mapping_existence(field, source_mapping, unmapped)
 
         if self.is_strict_mapping and unmapped:
             raise StrictPlatformException(
@@ -203,6 +229,13 @@ def check_fields_mapping_existence(self, field_tokens: list[Field], source_mappi
 
         return unmapped
 
+    @staticmethod
+    def _check_field_mapping_existence(field: Field, source_mapping: SourceMapping, unmapped: list[str]) -> None:
+        generic_field_name = field.get_generic_field_name(source_mapping.source_id)
+        mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
+        if not mapped_field and field.source_name not in unmapped:
+            unmapped.append(field.source_name)
+
     @staticmethod
     def map_field(field: Field, source_mapping: SourceMapping) -> list[str]:
         generic_field_name = field.get_generic_field_name(source_mapping.source_id)

uncoder-core/app/translator/core/mixins/rule.py

@@ -42,7 +42,7 @@ def parse_mitre_attack(self, tags: list[str]) -> MitreInfoContainer:
             tag = tag.lower()
             if tag.startswith("attack."):
                 tag = tag[7::]
-            if tag.startswith("t"):
+            if tag[-1].isdigit():
                 parsed_techniques.append(tag)
             else:
                 parsed_tactics.append(tag)

uncoder-core/app/translator/core/models/query_container.py

@@ -65,6 +65,8 @@ def __init__(
         date: Optional[str] = None,
         output_table_fields: Optional[list[Field]] = None,
         query_fields: Optional[list[Field]] = None,
+        function_fields: Optional[list[Field]] = None,
+        function_fields_map: Optional[dict[str, list[Field]]] = None,
         license_: Optional[str] = None,
         severity: Optional[str] = None,
         references: Optional[list[str]] = None,
@@ -76,7 +78,7 @@ def __init__(
         parsed_logsources: Optional[dict] = None,
         timeframe: Optional[timedelta] = None,
         query_period: Optional[timedelta] = None,
-        mitre_attack: MitreInfoContainer = MitreInfoContainer(),
+        mitre_attack: Optional[MitreInfoContainer] = None,
         raw_metainfo_container: Optional[RawMetaInfoContainer] = None,
     ) -> None:
         self.id = id_ or str(uuid.uuid4())
@@ -90,19 +92,21 @@ def __init__(
         self.date = date or datetime.now().date().strftime("%Y-%m-%d")
         self.output_table_fields = output_table_fields or []
         self.query_fields = query_fields or []
+        self.function_fields = function_fields or []
+        self.function_fields_map = function_fields_map or {}
         self.license = license_ or "DRL 1.1"
         self.severity = severity or SeverityType.low
         self.references = references or []
         self.tags = tags or []
-        self.mitre_attack = mitre_attack or None
+        self.mitre_attack = mitre_attack or MitreInfoContainer()
         self.raw_mitre_attack = raw_mitre_attack or []
         self.status = status or "stable"
         self.false_positives = false_positives or []
         self._source_mapping_ids = source_mapping_ids or [DEFAULT_MAPPING_NAME]
         self.parsed_logsources = parsed_logsources or {}
         self.timeframe = timeframe
         self.query_period = query_period
-        self.raw_metainfo_container = raw_metainfo_container
+        self.raw_metainfo_container = raw_metainfo_container or RawMetaInfoContainer()
 
     @property
     def author_str(self) -> str:

uncoder-core/app/translator/core/parser.py

@@ -24,7 +24,7 @@
 from app.translator.core.exceptions.parser import TokenizerGeneralException
 from app.translator.core.functions import PlatformFunctions
 from app.translator.core.mapping import BasePlatformMappings, SourceMapping
-from app.translator.core.models.functions.base import Function
+from app.translator.core.models.functions.base import Function, ParsedFunctions
 from app.translator.core.models.platform_details import PlatformDetails
 from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer
 from app.translator.core.models.query_tokens.field import Field
@@ -51,6 +51,9 @@ def parse_raw_query(self, text: str, language: str) -> RawQueryContainer:
     def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer:
         raise NotImplementedError("Abstract method")
 
+    def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]:
+        raise NotImplementedError("Abstract method")
+
 
 class PlatformQueryParser(QueryParser, ABC):
     mappings: BasePlatformMappings = None
@@ -65,16 +68,19 @@ def get_query_tokens(self, query: str) -> list[QUERY_TOKEN_TYPE]:
     @staticmethod
     def get_field_tokens(
         query_tokens: list[QUERY_TOKEN_TYPE], functions: Optional[list[Function]] = None
-    ) -> list[Field]:
-        field_tokens = []
+    ) -> tuple[list[Field], list[Field], dict[str, list[Field]]]:
+        query_field_tokens = []
+        function_field_tokens = []
+        function_field_tokens_map = {}
         for token in query_tokens:
             if isinstance(token, (FieldField, FieldValue, FunctionValue)):
-                field_tokens.extend(token.fields)
+                query_field_tokens.extend(token.fields)
 
-        if functions:
-            field_tokens.extend([field for func in functions for field in func.fields])
+        for func in functions or []:
+            function_field_tokens.extend(func.fields)
+            function_field_tokens_map[func.name] = func.fields
 
-        return field_tokens
+        return query_field_tokens, function_field_tokens, function_field_tokens_map
 
     def get_source_mappings(
         self, field_tokens: list[Field], log_sources: dict[str, list[Union[int, str]]]
@@ -85,3 +91,8 @@ def get_source_mappings(
         )
         self.tokenizer.set_field_tokens_generic_names_map(field_tokens, source_mappings, self.mappings.default_mapping)
         return source_mappings
+
+    def get_source_mapping_ids_by_logsources(self, query: str) -> Optional[list[str]]:
+        _, parsed_logsources, _ = self._parse_query(query=query)
+        if parsed_logsources:
+            return self.mappings.get_source_mappings_by_log_sources(parsed_logsources)

uncoder-core/app/translator/core/render.py

@@ -428,14 +428,18 @@ def _generate_from_tokenized_query_container_by_source_mapping(
         self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping
     ) -> str:
         unmapped_fields = self.mappings.check_fields_mapping_existence(
-            query_container.meta_info.query_fields, source_mapping
+            query_container.meta_info.query_fields,
+            query_container.meta_info.function_fields_map,
+            self.platform_functions.manager.supported_render_names,
+            source_mapping,
         )
         rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping)
         prefix = self.generate_prefix(source_mapping.log_source_signature, rendered_functions.rendered_prefix)
 
         if source_mapping.raw_log_fields:
             defined_raw_log_fields = self.generate_raw_log_fields(
-                fields=query_container.meta_info.query_fields, source_mapping=source_mapping
+                fields=query_container.meta_info.query_fields + query_container.meta_info.function_fields,
+                source_mapping=source_mapping,
             )
             prefix += f"\n{defined_raw_log_fields}"
         query = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping)

uncoder-core/app/translator/mappings/platforms/carbonblack/default.yml

@@ -0,0 +1,2 @@
+platform: CarbonBlack
+source: default

uncoder-core/app/translator/mappings/platforms/carbonblack/linux_dns_query.yml

@@ -0,0 +1,8 @@
+platform: CarbonBlack
+source: linux_dns_query
+
+
+field_mapping:
+  User:
+    - childproc_username
+    - process_username

uncoder-core/app/translator/mappings/platforms/carbonblack/linux_network_connection.yml

@@ -0,0 +1,9 @@
+platform: CarbonBlack
+source: linux_network_connection
+
+
+field_mapping:
+  DestinationHostname:
+    - netconn_domain
+    - netconn_proxy_domain
+  DestinationPort: netconn_port

uncoder-core/app/translator/mappings/platforms/carbonblack/macos_dns_query.yml

@@ -0,0 +1,8 @@
+platform: CarbonBlack
+source: macos_dns_query
+
+
+field_mapping:
+  User:
+    - childproc_username
+    - process_username