From d1c270999173570d05919c02e69b63f9e3b38c2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cleverton=20Guimar=C3=A3es?= Date: Mon, 11 Aug 2025 11:56:36 -0300 Subject: [PATCH 1/5] #113 Yield data in paginated requests + Added venv in .gitignore + Created _yield_paginated_request in apihelper & organize imports + Created yield_all in applications #114 Entities as methods responses + Created applications_entity in order to use with Generics on yield --- .gitignore | 8 ++ veracode_api_py/apihelper.py | 45 ++++++++-- veracode_api_py/applications.py | 10 +++ veracode_api_py/models/applications_entity.py | 88 +++++++++++++++++++ 4 files changed, 143 insertions(+), 8 deletions(-) create mode 100644 veracode_api_py/models/applications_entity.py diff --git a/.gitignore b/.gitignore index b94994e..628b722 100644 --- a/.gitignore +++ b/.gitignore @@ -152,3 +152,11 @@ dmypy.json cython_debug/ .DS_Store helpers/__pycache__/api.cpython-37.pyc + +# Python virtual environments +pyvenv.cfg +bin/ +lib/ +lib64/ +include/ +share/ diff --git a/veracode_api_py/apihelper.py b/veracode_api_py/apihelper.py index b59cd16..44e680a 100644 --- a/veracode_api_py/apihelper.py +++ b/veracode_api_py/apihelper.py @@ -1,22 +1,20 @@ # apihelper.py - API class for making network calls -import requests import logging -import json import time -from requests.adapters import HTTPAdapter +from typing import Generic, Iterable, TypeVar -from veracode_api_signing.exceptions import VeracodeAPISigningException -from veracode_api_signing.plugin_requests import RequestsAuthPluginVeracodeHMAC +import requests +from requests.adapters import HTTPAdapter from veracode_api_signing.credentials import get_credentials +from veracode_api_signing.plugin_requests import RequestsAuthPluginVeracodeHMAC from veracode_api_signing.regions import get_region_for_api_credential -from .exceptions import VeracodeAPIError -from .log import VeracodeLog as vlog from .constants import Constants +from .exceptions import VeracodeAPIError logger = logging.getLogger(__name__) - +T = TypeVar('T') class APIHelper(): api_key_id = None @@ -151,6 +149,37 @@ def _rest_paged_request(self, uri, method, element, params=None,fullresponse=Fal else: return all_data + def _yield_paginated_request(self, uri, method, entity: Generic[T], params=None) -> Iterable[T]: + """Paginates the request and yields the entities. + + Args: + uri: The URI to request. + method: The HTTP method to use. + entity: The entity to yield. + params: The parameters to pass to the request. + fullresponse: Whether to return the full response. + + Returns: + An iterable of the entities. + """ + + element = entity._element + + page = 0 + more_pages = True + + while more_pages: + params['page'] = page + page_data = self._rest_request(uri, method, params) + total_pages = page_data.get('page', {}).get('total_pages', 0) + data_page = page_data.get('_embedded', {}).get(element, []) + for data in data_page: + yield entity(**data) + + page += 1 + more_pages = page < total_pages + + def _xml_request(self, url, method, params=None, files=None): # base request method for XML APIs, handles what little error handling there is around these APIs if method not in ["GET", "POST"]: diff --git a/veracode_api_py/applications.py b/veracode_api_py/applications.py index 1339612..bc5eb0c 100644 --- a/veracode_api_py/applications.py +++ b/veracode_api_py/applications.py @@ -6,8 +6,18 @@ from .apihelper import APIHelper from .constants import Constants +from .models.applications_entity import ApplicationsEntity + class Applications(): + def yield_all(self, policy_check_after=None): + if policy_check_after == None: + params={} + else: + params={"policy_compliance_checked_after": policy_check_after} + + return APIHelper()._yield_paginated_request('appsec/v1/applications',"GET", ApplicationsEntity, params=params) + def get_all(self,policy_check_after=None): if policy_check_after == None: params={} diff --git a/veracode_api_py/models/applications_entity.py b/veracode_api_py/models/applications_entity.py new file mode 100644 index 0000000..e636958 --- /dev/null +++ b/veracode_api_py/models/applications_entity.py @@ -0,0 +1,88 @@ +from datetime import datetime +from typing import List, Optional, Dict, Any + +class Scan: + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + scan_type: str + status: str + modified_date: datetime + scan_url: str + internal_status: str + +class BusinessUnit: + id: int + name: str + guid: str + +class BusinessOwner: + name: str + email: str + +class Policy: + guid: str + name: str + is_default: bool + policy_compliance_status: str + +class Team: + team_id: int + team_name: str + guid: str + +class CustomField: + name: str + value: str + +class Settings: + nextday_consultation_allowed: bool + static_scan_xpa_or_dpa: bool + dynamic_scan_approval_not_required: bool + sca_enabled: bool + static_scan_xpp_enabled: bool + +class Profile: + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + name: str + tags: str + business_unit: BusinessUnit + business_owners: List[BusinessOwner] + archer_app_name: Optional[str] + enterprise_id: int + policies: List[Policy] + teams: List[Team] + custom_fields: List[CustomField] + description: str + settings: Settings + git_repo_url: Optional[str] + vendor_rescan: bool + business_criticality: str + +class ApplicationsEntity: + _element: str = 'applications' + id: int + oid: int + last_completed_scan_date: datetime + guid: str + created: datetime + modified: datetime + alt_org_id: int + app_profile_url: str + scans: List[Scan] + last_policy_compliance_check_date: datetime + profile: Profile + results_url: str + + def __init__(self, **kwargs): + for key, value in kwargs.items(): + if key == 'profile': + self.profile = Profile(**value) + elif key == 'scans': + self.scans = [Scan(**s) for s in value] + else: + setattr(self, key, value) From 1bd2ef2316baa817cde8538db76af83f780d1e5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cleverton=20Guimar=C3=A3es?= Date: Mon, 11 Aug 2025 12:03:58 -0300 Subject: [PATCH 2/5] #113 Added docs --- docs/applications.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/applications.md b/docs/applications.md index fe4ed39..96d7303 100644 --- a/docs/applications.md +++ b/docs/applications.md @@ -4,6 +4,7 @@ The following methods call Veracode REST APIs and return JSON. ## Applications +- `Applications().yield_all(policy_check_after(opt))` : get a list of Veracode applications. If provided, returns only applications that have a policy check date on or after `policy_check_after` (format is `yyyy-mm-dd`). - `Applications().get_all(policy_check_after(opt))` : get a list of Veracode applications (JSON format). If provided, returns only applications that have a policy check date on or after `policy_check_after` (format is `yyyy-mm-dd`). - `Applications().get(guid(opt),legacy_id(opt))`: get information for a single Veracode application using either the `guid` or the `legacy_id` (integer). - `Applications().get_by_name(name)`: get list of applications whose names contain the search string `name`. From ac5bb63b07efafbdc0fdc77a8cc674cd26acb80b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cleverton=20Guimar=C3=A3es?= Date: Mon, 11 Aug 2025 17:38:09 -0300 Subject: [PATCH 3/5] Using dacite to parse API response with generics and improved type annotation --- requirements.txt | 3 +- veracode_api_py/apihelper.py | 24 +++++++----- veracode_api_py/applications.py | 3 +- veracode_api_py/models/applications_entity.py | 39 ++++++++++--------- 4 files changed, 40 insertions(+), 29 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7a1145d..31c776d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ requests==2.32.4 veracode-api-signing>=24.11.0 Pygments>= 2.9.0 idna>=3.7 -certifi>=2024.7.4 \ No newline at end of file +certifi>=2024.7.4 +dacite>=1.8.0 \ No newline at end of file diff --git a/veracode_api_py/apihelper.py b/veracode_api_py/apihelper.py index 44e680a..249c006 100644 --- a/veracode_api_py/apihelper.py +++ b/veracode_api_py/apihelper.py @@ -1,10 +1,11 @@ # apihelper.py - API class for making network calls - import logging import time -from typing import Generic, Iterable, TypeVar +from datetime import datetime +from typing import Iterable, TypeVar, Type import requests +from dacite import from_dict, Config from requests.adapters import HTTPAdapter from veracode_api_signing.credentials import get_credentials from veracode_api_signing.plugin_requests import RequestsAuthPluginVeracodeHMAC @@ -16,6 +17,7 @@ logger = logging.getLogger(__name__) T = TypeVar('T') + class APIHelper(): api_key_id = None api_key_secret = None @@ -149,7 +151,7 @@ def _rest_paged_request(self, uri, method, element, params=None,fullresponse=Fal else: return all_data - def _yield_paginated_request(self, uri, method, entity: Generic[T], params=None) -> Iterable[T]: + def _yield_paginated_request(self, uri, method, entity: Type[T], params=None) -> Iterable[T]: """Paginates the request and yields the entities. Args: @@ -162,9 +164,9 @@ def _yield_paginated_request(self, uri, method, entity: Generic[T], params=None) Returns: An iterable of the entities. """ - + element = entity._element - + page = 0 more_pages = True @@ -174,12 +176,16 @@ def _yield_paginated_request(self, uri, method, entity: Generic[T], params=None) total_pages = page_data.get('page', {}).get('total_pages', 0) data_page = page_data.get('_embedded', {}).get(element, []) for data in data_page: - yield entity(**data) + config = Config( + type_hooks={ + datetime: lambda x: datetime.fromisoformat(x.replace('Z', '+00:00')) if isinstance(x, str) else x + } + ) + yield from_dict(data_class=entity, data=data, config=config) page += 1 more_pages = page < total_pages - def _xml_request(self, url, method, params=None, files=None): # base request method for XML APIs, handles what little error handling there is around these APIs if method not in ["GET", "POST"]: @@ -189,14 +195,14 @@ def _xml_request(self, url, method, params=None, files=None): session = requests.Session() session.mount(self.baseurl, HTTPAdapter(max_retries=3)) request = requests.Request(method, url, params=params, files=files, - auth=RequestsAuthPluginVeracodeHMAC(), headers=self._prepare_headers(method,'xml')) + auth=RequestsAuthPluginVeracodeHMAC(), headers=self._prepare_headers(method, 'xml')) prepared_request = request.prepare() r = session.send(prepared_request) if 200 <= r.status_code <= 299: if r.status_code == 204: # retry after wait time.sleep(self.retry_seconds) - return self._xml_request(url,method,params) + return self._xml_request(url, method, params) elif r.content is None: logger.debug("HTTP response body empty:\r\n{}\r\n{}\r\n{}\r\n\r\n{}\r\n{}\r\n{}\r\n" .format(r.request.url, r.request.headers, r.request.body, r.status_code, r.headers, diff --git a/veracode_api_py/applications.py b/veracode_api_py/applications.py index bc5eb0c..888dc91 100644 --- a/veracode_api_py/applications.py +++ b/veracode_api_py/applications.py @@ -1,6 +1,7 @@ #applications.py - API class for Applications API calls import json +from typing import Iterable from urllib import parse from uuid import UUID @@ -10,7 +11,7 @@ class Applications(): - def yield_all(self, policy_check_after=None): + def yield_all(self, policy_check_after=None) -> Iterable[ApplicationsEntity]: if policy_check_after == None: params={} else: diff --git a/veracode_api_py/models/applications_entity.py b/veracode_api_py/models/applications_entity.py index e636958..ea4e37c 100644 --- a/veracode_api_py/models/applications_entity.py +++ b/veracode_api_py/models/applications_entity.py @@ -1,41 +1,52 @@ +from dataclasses import dataclass from datetime import datetime -from typing import List, Optional, Dict, Any +from typing import List, Optional -class Scan: - def __init__(self, **kwargs): - for key, value in kwargs.items(): - setattr(self, key, value) +@dataclass +class Scan: scan_type: str status: str modified_date: datetime scan_url: str internal_status: str + +@dataclass class BusinessUnit: id: int name: str guid: str + +@dataclass class BusinessOwner: name: str email: str + +@dataclass class Policy: guid: str name: str is_default: bool policy_compliance_status: str + +@dataclass class Team: team_id: int team_name: str guid: str + +@dataclass class CustomField: name: str value: str + +@dataclass class Settings: nextday_consultation_allowed: bool static_scan_xpa_or_dpa: bool @@ -43,11 +54,9 @@ class Settings: sca_enabled: bool static_scan_xpp_enabled: bool -class Profile: - def __init__(self, **kwargs): - for key, value in kwargs.items(): - setattr(self, key, value) +@dataclass +class Profile: name: str tags: str business_unit: BusinessUnit @@ -63,8 +72,9 @@ def __init__(self, **kwargs): vendor_rescan: bool business_criticality: str + +@dataclass class ApplicationsEntity: - _element: str = 'applications' id: int oid: int last_completed_scan_date: datetime @@ -78,11 +88,4 @@ class ApplicationsEntity: profile: Profile results_url: str - def __init__(self, **kwargs): - for key, value in kwargs.items(): - if key == 'profile': - self.profile = Profile(**value) - elif key == 'scans': - self.scans = [Scan(**s) for s in value] - else: - setattr(self, key, value) + _element: str = 'applications' From 6cec094e0d53bfca4b6cc689485f8cff8aaa5c61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cleverton=20Guimar=C3=A3es?= Date: Tue, 26 Aug 2025 17:59:49 -0300 Subject: [PATCH 4/5] Yield for findings --- veracode_api_py/findings.py | 26 ++++++++++ veracode_api_py/models/findings_entity.py | 59 +++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 veracode_api_py/models/findings_entity.py diff --git a/veracode_api_py/findings.py b/veracode_api_py/findings.py index c48f0ad..d413c68 100644 --- a/veracode_api_py/findings.py +++ b/veracode_api_py/findings.py @@ -1,14 +1,40 @@ #findings.py - API class for Findings API and related calls import json +from typing import Iterable from uuid import UUID +from veracode_api_py.models.findings_entity import FindingsEntity + from .apihelper import APIHelper LINE_NUMBER_SLOP = 3 #adjust to allow for line number movement class Findings(): + def yield_all(self, app: UUID, scantype='STATIC', annot='TRUE', request_params=None, sandbox: UUID = None) -> Iterable[FindingsEntity]: + if request_params == None: + request_params = {} + + scantypes = "" + scantype = scantype.split(',') + for st in scantype: + if st in ['STATIC', 'DYNAMIC', 'MANUAL', 'SCA']: + if len(scantypes) > 0: + scantypes += "," + scantypes += st + if len(scantypes) > 0: + request_params['scan_type'] = scantypes + # note that scantype='ALL' will result in no scan_type parameter as in API + + request_params['include_annot'] = annot + + if sandbox != None: + request_params['context'] = sandbox + + uri = "appsec/v2/applications/{}/findings".format(app) + return APIHelper()._yield_paginated_request(uri, "GET", FindingsEntity, request_params) + def get_findings(self,app: UUID,scantype='STATIC',annot='TRUE',request_params=None,sandbox: UUID=None): #Gets a list of findings for app using the Veracode Findings API if request_params == None: diff --git a/veracode_api_py/models/findings_entity.py b/veracode_api_py/models/findings_entity.py new file mode 100644 index 0000000..4c87baf --- /dev/null +++ b/veracode_api_py/models/findings_entity.py @@ -0,0 +1,59 @@ +from dataclasses import dataclass +from datetime import datetime +from typing import Optional + + +@dataclass +class CWE: + id: int + name: str + href: str + + +@dataclass +class FindingCategory: + id: int + name: str + href: str + + +@dataclass +class FindingDetails: + severity: int + cwe: CWE + file_path: str + file_name: str + module: str + relative_location: int + finding_category: FindingCategory + procedure: str + exploitability: int + attack_vector: str + file_line_number: int + + +@dataclass +class FindingStatus: + first_found_date: datetime + status: str + resolution: str + mitigation_review_status: str + new: bool + resolution_status: str + last_seen_date: datetime + + +@dataclass +class FindingsEntity: + issue_id: int + scan_type: str + description: str + count: int + context_type: str + context_guid: str + violates_policy: bool + finding_status: FindingStatus + finding_details: FindingDetails + build_id: int + + _element: str = 'findings' From 4b8649f2c09476fd7d3480a087e0ce8119c4e608 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cleverton=20Guimar=C3=A3es?= Date: Thu, 4 Sep 2025 13:40:15 -0300 Subject: [PATCH 5/5] Allow optional types --- veracode_api_py/models/applications_entity.py | 96 +++++++++---------- veracode_api_py/models/findings_entity.py | 68 ++++++------- 2 files changed, 82 insertions(+), 82 deletions(-) diff --git a/veracode_api_py/models/applications_entity.py b/veracode_api_py/models/applications_entity.py index ea4e37c..52fa19b 100644 --- a/veracode_api_py/models/applications_entity.py +++ b/veracode_api_py/models/applications_entity.py @@ -5,87 +5,87 @@ @dataclass class Scan: - scan_type: str - status: str - modified_date: datetime - scan_url: str - internal_status: str + scan_type: Optional[str] + status: Optional[str] + modified_date: Optional[datetime] + scan_url: Optional[str] + internal_status: Optional[str] @dataclass class BusinessUnit: - id: int - name: str - guid: str + id: Optional[int] + name: Optional[str] + guid: Optional[str] @dataclass class BusinessOwner: - name: str - email: str + name: Optional[str] + email: Optional[str] @dataclass class Policy: - guid: str - name: str - is_default: bool - policy_compliance_status: str + guid: Optional[str] + name: Optional[str] + is_default: Optional[bool] + policy_compliance_status: Optional[str] @dataclass class Team: - team_id: int - team_name: str - guid: str + team_id: Optional[int] + team_name: Optional[str] + guid: Optional[str] @dataclass class CustomField: - name: str - value: str + name: Optional[str] + value: Optional[str] @dataclass class Settings: - nextday_consultation_allowed: bool - static_scan_xpa_or_dpa: bool - dynamic_scan_approval_not_required: bool - sca_enabled: bool - static_scan_xpp_enabled: bool + nextday_consultation_allowed: Optional[bool] + static_scan_xpa_or_dpa: Optional[bool] + dynamic_scan_approval_not_required: Optional[bool] + sca_enabled: Optional[bool] + static_scan_xpp_enabled: Optional[bool] @dataclass class Profile: - name: str - tags: str - business_unit: BusinessUnit - business_owners: List[BusinessOwner] + name: Optional[str] + tags: Optional[str] + business_unit: Optional[BusinessUnit] + business_owners: Optional[List[BusinessOwner]] archer_app_name: Optional[str] - enterprise_id: int - policies: List[Policy] - teams: List[Team] - custom_fields: List[CustomField] - description: str - settings: Settings + enterprise_id: Optional[int] + policies: Optional[List[Policy]] + teams: Optional[List[Team]] + custom_fields: Optional[List[CustomField]] + description: Optional[str] + settings: Optional[Settings] git_repo_url: Optional[str] - vendor_rescan: bool - business_criticality: str + vendor_rescan: Optional[bool] + business_criticality: Optional[str] @dataclass class ApplicationsEntity: - id: int - oid: int - last_completed_scan_date: datetime - guid: str - created: datetime - modified: datetime - alt_org_id: int - app_profile_url: str - scans: List[Scan] - last_policy_compliance_check_date: datetime - profile: Profile - results_url: str + id: Optional[int] + oid: Optional[int] + last_completed_scan_date: Optional[datetime] + guid: Optional[str] + created: Optional[datetime] + modified: Optional[datetime] + alt_org_id: Optional[int] + app_profile_url: Optional[str] + scans: Optional[List[Scan]] + last_policy_compliance_check_date: Optional[datetime] + profile: Optional[Profile] + results_url: Optional[str] _element: str = 'applications' diff --git a/veracode_api_py/models/findings_entity.py b/veracode_api_py/models/findings_entity.py index 4c87baf..be5c1ad 100644 --- a/veracode_api_py/models/findings_entity.py +++ b/veracode_api_py/models/findings_entity.py @@ -5,55 +5,55 @@ @dataclass class CWE: - id: int - name: str - href: str + id: Optional[int] + name: Optional[str] + href: Optional[str] @dataclass class FindingCategory: - id: int - name: str - href: str + id: Optional[int] + name: Optional[str] + href: Optional[str] @dataclass class FindingDetails: - severity: int - cwe: CWE - file_path: str - file_name: str - module: str - relative_location: int - finding_category: FindingCategory - procedure: str - exploitability: int - attack_vector: str - file_line_number: int + severity: Optional[int] + cwe: Optional[CWE] + file_path: Optional[str] + file_name: Optional[str] + module: Optional[str] + relative_location: Optional[int] + finding_category: Optional[FindingCategory] + procedure: Optional[str] + exploitability: Optional[int] + attack_vector: Optional[str] + file_line_number: Optional[int] @dataclass class FindingStatus: - first_found_date: datetime - status: str - resolution: str - mitigation_review_status: str - new: bool - resolution_status: str - last_seen_date: datetime + first_found_date: Optional[datetime] + status: Optional[str] + resolution: Optional[str] + mitigation_review_status: Optional[str] + new: Optional[bool] + resolution_status: Optional[str] + last_seen_date: Optional[datetime] @dataclass class FindingsEntity: - issue_id: int - scan_type: str - description: str - count: int - context_type: str - context_guid: str - violates_policy: bool - finding_status: FindingStatus - finding_details: FindingDetails - build_id: int + issue_id: Optional[int] + scan_type: Optional[str] + description: Optional[str] + count: Optional[int] + context_type: Optional[str] + context_guid: Optional[str] + violates_policy: Optional[bool] + finding_status: Optional[FindingStatus] + finding_details: Optional[FindingDetails] + build_id: Optional[int] _element: str = 'findings'