From 93cd91302d54704ea922da72eac6ea960ac840d3 Mon Sep 17 00:00:00 2001 From: Sermet Pekin Date: Sun, 21 Apr 2024 22:08:37 +0300 Subject: [PATCH 1/4] user req --- .../index_requests/get_series_indexes.py | 43 +- .../user_requests/Data_processor.py | 35 ++ .../user_requests/Proxy_manager.py | 29 ++ .../user_requests/Request_config.py | 140 ++++++ .../index_requests/user_requests/Serialize.py | 23 + .../user_requests/Url_builder.py | 53 +++ .../user_requests/Url_series.py | 17 + .../user_requests/User_req_typings.py | 10 + .../user_requests/User_request_utils.py | 62 +++ .../index_requests/user_requests/__init__.py | 13 + .../user_requests/user_requests.py | 447 ------------------ .../user_requests/user_requests_2.py | 134 ------ evdspy/EVDSlocal/tests/test_user_requests.py | 16 +- evdspy/__init__.py | 13 +- 14 files changed, 438 insertions(+), 597 deletions(-) create mode 100644 evdspy/EVDSlocal/index_requests/user_requests/Data_processor.py create mode 100644 evdspy/EVDSlocal/index_requests/user_requests/Proxy_manager.py create mode 100644 evdspy/EVDSlocal/index_requests/user_requests/Request_config.py create mode 100644 evdspy/EVDSlocal/index_requests/user_requests/Serialize.py create mode 100644 evdspy/EVDSlocal/index_requests/user_requests/Url_builder.py create mode 100644 evdspy/EVDSlocal/index_requests/user_requests/Url_series.py create mode 100644 evdspy/EVDSlocal/index_requests/user_requests/User_req_typings.py create mode 100644 evdspy/EVDSlocal/index_requests/user_requests/User_request_utils.py delete mode 100644 evdspy/EVDSlocal/index_requests/user_requests/user_requests.py delete mode 100644 evdspy/EVDSlocal/index_requests/user_requests/user_requests_2.py diff --git a/evdspy/EVDSlocal/index_requests/get_series_indexes.py b/evdspy/EVDSlocal/index_requests/get_series_indexes.py index 8e7b8f3..73e91bc 100644 --- a/evdspy/EVDSlocal/index_requests/get_series_indexes.py +++ b/evdspy/EVDSlocal/index_requests/get_series_indexes.py @@ -1,19 +1,31 @@ - from typing import Union, Any, Optional + import pandas as pd -from evdspy.EVDSlocal.config.apikey_class import ApikeyClass -from evdspy.EVDSlocal.index_requests.get_series_indexes_utils import default_start_date_fnc, default_end_date_fnc -from evdspy.EVDSlocal.index_requests.user_requests.user_requests import ProxyManager, UrlBuilder, ApiRequester, \ - DataProcessor, RequestConfig +from evdspy.EVDSlocal.index_requests.user_requests.Request_config import RequestConfig def initial_api_process_when_given(api_key: Optional[str] = None) -> None: + from evdspy.EVDSlocal.config.apikey_class import ApikeyClass + from evdspy.EVDSlocal.index_requests.get_series_indexes_utils import default_start_date_fnc, default_end_date_fnc + + from evdspy.EVDSlocal.index_requests.user_requests import RequestConfig, ProxyManager, \ + UrlBuilder, DataProcessor + from evdspy.EVDSlocal.index_requests.user_requests.Api_requester import ApiRequester + + if api_key is None: return if ApikeyClass().get_valid_api_key(check=False) is False: from evdspy.EVDSlocal.initial.load_commands_cmds_to_load import save_apikey save_apikey(api_key) + +from evdspy.EVDSlocal.index_requests.get_series_indexes_utils import ( + default_start_date_fnc, + default_end_date_fnc, + correct_types, +) + def get_series( index: Union[str, tuple[Any, ...]], start_date: str = default_start_date_fnc(), @@ -67,6 +79,13 @@ def get_series( ValueError If an invalid API key is provided or required parameters are missing. """ + from evdspy.EVDSlocal.config.apikey_class import ApikeyClass + from evdspy.EVDSlocal.index_requests.get_series_indexes_utils import default_start_date_fnc, default_end_date_fnc + + from evdspy.EVDSlocal.index_requests.user_requests import RequestConfig, ProxyManager, \ + UrlBuilder, DataProcessor + from evdspy.EVDSlocal.index_requests.user_requests.Api_requester import ApiRequester + # ............initial_api_process_when_given............... initial_api_process_when_given(api_key) # ............RequestConfig................................ @@ -89,19 +108,25 @@ def get_series( # ............DataProcessor................................ data_processor = DataProcessor(api_requester()) return data_processor() + + def test_get_series2(capsys): with capsys.disabled(): # setup() df = get_series( - "TP.ODEMGZS.BDTTOPLAM", - cache=False + "TP.ODEMGZS.BDTTOPLAM", + cache=False ) assert isinstance(df, pd.DataFrame) + + def t_stream(): import streamlit as st df = get_series("TP.ODEMGZS.BDTTOPLAM", cache=True) st.write(df) + + __all__ = ( - 'get_series', -) \ No newline at end of file + 'get_series', +) diff --git a/evdspy/EVDSlocal/index_requests/user_requests/Data_processor.py b/evdspy/EVDSlocal/index_requests/user_requests/Data_processor.py new file mode 100644 index 0000000..7198bb6 --- /dev/null +++ b/evdspy/EVDSlocal/index_requests/user_requests/Data_processor.py @@ -0,0 +1,35 @@ +# ....................................................................... DataProcessor +import traceback +from typing import Any + +import pandas as pd + +from evdspy.EVDSlocal.index_requests.index_util_funcs import make_df_float, json_to_df +from evdspy.EVDSlocal.index_requests.user_requests.User_req_typings import T_maybeDf + + +class DataProcessor: + def __init__(self, data: Any): + self.data = data + + def process_to_dataframe(self) -> T_maybeDf: + if self.data is False: + return False + try: + df = json_to_df(self.data) + except Exception as e: + print(e) + traceback.print_exc() + return None + if isinstance(df, pd.DataFrame): + df = make_df_float(df) + return df + + def __call__(self, *args, **kwargs) -> T_maybeDf: + return self.process_to_dataframe() + + +def test_DataProcessor(capsys): + with capsys.disabled(): + d = DataProcessor(False) + print(d) diff --git a/evdspy/EVDSlocal/index_requests/user_requests/Proxy_manager.py b/evdspy/EVDSlocal/index_requests/user_requests/Proxy_manager.py new file mode 100644 index 0000000..5694e18 --- /dev/null +++ b/evdspy/EVDSlocal/index_requests/user_requests/Proxy_manager.py @@ -0,0 +1,29 @@ +# ....................................................................... ProxyManager +from dataclasses import dataclass +from typing import Optional, Any + + +@dataclass +class ProxyManager: + proxy: Optional[str] = None + proxies: Optional[dict[Any, Any]] = None + + def get_proxies(self) -> Optional[dict[Any, Any]]: + if self.proxies is None: + if self.proxy is None: + proxies = None + else: + proxies = self.get_proxies_helper() + else: + proxies = self.proxies + return proxies + + def get_proxies_helper(self) -> Optional[dict[Any, Any]]: + if self.proxy is None: + return None + proxy = self.proxy + proxies = { + 'http': proxy, + 'https': proxy, + } + return proxies diff --git a/evdspy/EVDSlocal/index_requests/user_requests/Request_config.py b/evdspy/EVDSlocal/index_requests/user_requests/Request_config.py new file mode 100644 index 0000000..ecf4970 --- /dev/null +++ b/evdspy/EVDSlocal/index_requests/user_requests/Request_config.py @@ -0,0 +1,140 @@ +# ................................................................... RequestConfig +from dataclasses import dataclass +from typing import Union, Optional + +# from evdspy import default_start_date_fnc, default_end_date_fnc +from evdspy.EVDSlocal.components.excel_class import replace_all +from evdspy.EVDSlocal.index_requests.get_series_indexes_utils import freq_enum, correct_types, AggregationType, \ + Formulas, default_start_date_fnc, default_end_date_fnc +from evdspy.EVDSlocal.index_requests.user_requests.Serialize import Serialize +from evdspy.EVDSlocal.index_requests.user_requests.User_req_typings import T_tuple_str_int_None, T_str_tuple_None, \ + T_str_int_None +from evdspy.EVDSlocal.index_requests.user_requests.User_request_utils import UserRequestUtils + + +@dataclass +class RequestConfig(Serialize): + index: Union[str, tuple[str, ...], list[str, ...]] + start_date: str = default_start_date_fnc() + end_date: str = default_end_date_fnc() + frequency: T_str_int_None = None + formulas: T_tuple_str_int_None = None + aggregation: T_str_tuple_None = None + cache: bool = False + cache_name: str = "" + initial_index = None + # series_url_instance: SeriesUrlClass = SeriesUrlClass() + """ + Represents a user request for fetching data series from a specified source. + This class encapsulates the details necessary to construct and execute a data retrieval request, + handling various configurations like date ranges, frequency, and data aggregation types. It + also manages proxy settings and caching mechanisms to optimize data retrieval operations. + Attributes: + index (Union[str, Tuple[str]]): The identifier(s) for the data series to fetch. + start_date (str): The start date for the data retrieval in 'DD-MM-YYYY' format. + end_date (str): The end date for the data retrieval in 'DD-MM-YYYY' format. + frequency (Union[str, int, None]): The frequency at which data should be retrieved. + formulas (Union[str, int, Tuple[str, int], None]): The calculation types to apply to the data. + aggregation (Union[str, Tuple[str], None]): The aggregation methods to apply to the data. + cache (bool): Enables or disables caching of the request results. + cache_name (str): The name under which to store the cached results, if caching is enabled. + series_url_instance (SeriesUrlClass): An instance of SeriesUrlClass to handle URL creation. + Methods: + __post_init__: Initializes further attributes and performs checks after the basic initialization. + Raises: + ValueError: If the index is in an invalid format or necessary attributes are missing. + """ + + def check_type(self): + if UserRequestUtils.looks_like_datagroup(self.initial_index): + return "datagroup" + return "series" + + def __post_init__(self): + self.initial_index = self.index + self.correct_index() + self.correct_formulas_aggr() + self.check() + + def upper_index(self) -> None: + def upper(string: str) -> str: + if UserRequestUtils.looks_like_datagroup(self.initial_index): + return string + return string.upper() + + self.index = tuple(map(upper, self.index)) + + def clean(self) -> None: + self.index = tuple(map(UserRequestUtils.clean_chars, self.index)) + + def correct_index(self): + if "\n" in self.index or "\t" in self.index: + self.index = self.template_to_tuple(self.index) + self.index = tuple([self.index]) if not isinstance(self.index, (tuple, list,)) else self.index + self.clean() + self.upper_index() + self.check_index() + + def correct_formulas_aggr(self): + self.formulas = self.correct_type_to_tuple(self.formulas) + self.aggregation = self.correct_type_to_tuple(self.aggregation) + + def correct_type_to_tuple(self, value: any) -> Optional[tuple]: + if value is None: + return None + if isinstance(value, (list, tuple)): + if len(value) == len(self.index): + return value + if isinstance(value, (str,)): + return tuple(value for _ in self.index) + return tuple(value[0] for _ in self.index) + + @staticmethod + def template_to_tuple(index: str) -> tuple: + def clean(string: str): + return string.split("#")[0].strip() if len(string.split("#")) > 0 else None + + index = replace_all(index, "\t", "\n") + index_tuple = index.splitlines() + t = tuple(clean(x) for x in index_tuple) + return tuple(x for x in t if x is not None and len(x) > 3) + + def check_index(self) -> None: + if isinstance(self.index, (int, float,)): + raise ValueError("index must be a string ot tuple of string ") + + def freq_str(self) -> str: + if self.frequency is None: + return "" + if isinstance(self.frequency, int): + return f"&frequency={self.frequency}" + return freq_enum(self.frequency) + + def agr_form_type_to_str(self, value: Optional[tuple], part_name="aggregationTypes"): + if value is None: + return "" + value = tuple(map(str, value)) + string = "-".join(value) + return f"&{part_name}=" + string + + def aggregation_type_to_str(self) -> str: + self.aggregation = correct_types(self.aggregation, enum_class=AggregationType) + return self.agr_form_type_to_str(self.aggregation, "aggregationTypes") + + def formulas_to_str(self) -> str: + self.formulas = correct_types(self.formulas, enum_class=Formulas) + return self.agr_form_type_to_str(self.formulas, "formulas") + + def check(self) -> None: + if self.formulas is not None: + assert len(self.formulas) == len(self.index) + if self.aggregation is not None: + assert len(self.aggregation) == len(self.index) + if self.frequency is not None: + assert isinstance(self.frequency, (int, str,)) + + def create_series_part(self) -> str: + indexes = self.index + if isinstance(indexes, str): + indexes = tuple([indexes]) + return "-".join(indexes) diff --git a/evdspy/EVDSlocal/index_requests/user_requests/Serialize.py b/evdspy/EVDSlocal/index_requests/user_requests/Serialize.py new file mode 100644 index 0000000..1f4987a --- /dev/null +++ b/evdspy/EVDSlocal/index_requests/user_requests/Serialize.py @@ -0,0 +1,23 @@ +import json +from abc import ABC + + +# ....................................................................... Serialize +class Serialize(ABC): + """To check whether two Config Requests are perfect substitutes """ + + def serialize(self): + return json.dumps( + self, + default=lambda o: o.__dict__, + sort_keys=True, + indent=4) + + @property + def hash(self) -> str: + import hashlib + return str(int(hashlib.sha256(self.serialize().encode('utf-8')).hexdigest(), 16) % 10 ** 8) + + def __eq__(self, other): + return self.hash == other.hash + diff --git a/evdspy/EVDSlocal/index_requests/user_requests/Url_builder.py b/evdspy/EVDSlocal/index_requests/user_requests/Url_builder.py new file mode 100644 index 0000000..28857f5 --- /dev/null +++ b/evdspy/EVDSlocal/index_requests/user_requests/Url_builder.py @@ -0,0 +1,53 @@ +# from evdspy.EVDSlocal.index_requests.user_requests import UrlSeries, UrlDataGroup +from evdspy.EVDSlocal.index_requests.user_requests.Request_config import RequestConfig + + +# ....................................................................... UrlBuilder +class UrlBuilder: + def __init__(self, + config: RequestConfig, + url_type=None) -> None: + self.config = config + self.series_part = self.config.create_series_part() + if not url_type: + self.get_url_type() + self.alias = self.url_type.alias + + def get_url_type(self): + from evdspy.EVDSlocal.index_requests.user_requests import UrlSeries, UrlDataGroup + url_type = UrlSeries() + if self.config.check_type() == "datagroup": + url_type = UrlDataGroup() + self.url_type = url_type + + def create_url_for_series(cls) -> str: + domain = cls.domain + return f"{domain}/{cls.alias}{cls.series_part}&startDate={cls.config.start_date}&endDate={cls.config.end_date}&type=json" + + @property + def domain(self) -> str: + return self.url_type.domain + + @property + def basic_url(self) -> str: + config = self.config + return f"{self.domain}/{self.alias}{self.series_part}&startDate={config.start_date}&endDate={config.end_date}&type=json" + + @property + def url(self) -> str: + config = self.config + if config.frequency is None and config.aggregation is None and config.formulas is None: + return self.basic_url + """ config parts """ + formulas_str = config.formulas_to_str() + aggregation_type_str = config.aggregation_type_to_str() + freq_string = config.freq_str() + """...""" + parts = ( + f"{self.domain}/{self.alias}{self.series_part}{freq_string}{formulas_str}{aggregation_type_str}", + f"startDate={config.start_date}", + f"endDate={config.end_date}", + "type=json" + ) + return "&".join(parts) + diff --git a/evdspy/EVDSlocal/index_requests/user_requests/Url_series.py b/evdspy/EVDSlocal/index_requests/user_requests/Url_series.py new file mode 100644 index 0000000..e4c381d --- /dev/null +++ b/evdspy/EVDSlocal/index_requests/user_requests/Url_series.py @@ -0,0 +1,17 @@ +# ....................................................................... UrlSeries +class UrlSeries: + @property + def domain(self) -> str: + return "https://evds2.tcmb.gov.tr/service/evds" + + @property + def alias(self): + return "series=" + + + +# ....................................................................... UrlDataGroup +class UrlDataGroup(UrlSeries): + @property + def alias(self): + return "datagroup=" diff --git a/evdspy/EVDSlocal/index_requests/user_requests/User_req_typings.py b/evdspy/EVDSlocal/index_requests/user_requests/User_req_typings.py new file mode 100644 index 0000000..da64459 --- /dev/null +++ b/evdspy/EVDSlocal/index_requests/user_requests/User_req_typings.py @@ -0,0 +1,10 @@ +# typings .......................................................... typings +from typing import Union + +import pandas as pd + +T_str_int_None = Union[str, int, None] +T_str_tuple_None = Union[str, tuple[str], None] +T_tuple_str_int_None = Union[str, int, tuple[str], tuple[int], None] +T_maybeDf = Union[pd.DataFrame, bool, None] + diff --git a/evdspy/EVDSlocal/index_requests/user_requests/User_request_utils.py b/evdspy/EVDSlocal/index_requests/user_requests/User_request_utils.py new file mode 100644 index 0000000..69d70e9 --- /dev/null +++ b/evdspy/EVDSlocal/index_requests/user_requests/User_request_utils.py @@ -0,0 +1,62 @@ +# ................................................................. UserRequestUtils +# ........................................................................................... +from evdspy import setup + + + +from evdspy.EVDSlocal.requests_.real_requests import * +# ........................................................................................... +from typing import Union, Callable + + + +class UserRequestUtils: + @staticmethod + def looks_like_datagroup(string: Any): + if not isinstance(string, str): + return False + return 'bie_' in string + + @staticmethod + def clean_chars(string: str): + import re + if UserRequestUtils().looks_like_datagroup(string): + string = re.sub('[^0-9a-zA-Z._]+', '', string) + return string + string = string.replace("_", ".") + string = re.sub('[^0-9a-zA-Z.]+', '', string) + return string + + +def test_clean_chars(capsys): + with capsys.disabled(): + assert UserRequestUtils.clean_chars("AAA_BBB?*-()$? ") == "AAA.BBB" + assert UserRequestUtils.clean_chars("AAA..._BBB?*-()$? ") == "AAA....BBB" + assert UserRequestUtils.clean_chars("bie_BBB?*-()$? ") == "bie_BBB" + + +def test_cache_or_raw_fnc(capsys): + with capsys.disabled(): + setup() + fnc = lambda x: x ** 2 + f = cache_or_raw_fnc(fnc, True) + f2 = cache_or_raw_fnc(fnc, False) + assert f(3) == 9 + assert f2(3) == 9 + assert id(f2) != id(f) + + +# .................................................................................. + +def create_cache_version(fnc: Callable): + @MyCache().cache + def fnc_cache(*args, **kw): + return fnc(*args, **kw) + + return fnc_cache + + +def cache_or_raw_fnc(fnc, cache=False): + if not cache: + return fnc + return create_cache_version(fnc) diff --git a/evdspy/EVDSlocal/index_requests/user_requests/__init__.py b/evdspy/EVDSlocal/index_requests/user_requests/__init__.py index e69de29..c290875 100644 --- a/evdspy/EVDSlocal/index_requests/user_requests/__init__.py +++ b/evdspy/EVDSlocal/index_requests/user_requests/__init__.py @@ -0,0 +1,13 @@ +# from evdspy.EVDSlocal.index_requests.user_requests.Proxy_manager import ProxyManager, UrlBuilder, UrlSeries, \ +# ApiRequester, DataProcessor, RequestConfig + +from .Url_builder import * +from .Data_processor import * +from .Data_processor import * +from .Data_processor import * +from .Proxy_manager import * +from .Serialize import * +from .Url_series import * +from .User_req_typings import * +from .user_requests import * +from .Api_requester import ApiRequester \ No newline at end of file diff --git a/evdspy/EVDSlocal/index_requests/user_requests/user_requests.py b/evdspy/EVDSlocal/index_requests/user_requests/user_requests.py deleted file mode 100644 index 56dcd85..0000000 --- a/evdspy/EVDSlocal/index_requests/user_requests/user_requests.py +++ /dev/null @@ -1,447 +0,0 @@ -# ........................................................................................... -from evdspy import setup -from evdspy.EVDSlocal.components.excel_class import replace_all -from evdspy.EVDSlocal.index_requests.get_series_indexes_utils import ( - default_start_date_fnc, - default_end_date_fnc, correct_types, -) -from evdspy.EVDSlocal.utils.utils_test import get_api_key_while_testing, ApiClassWhileTesting -from evdspy.EVDSlocal.index_requests.index_util_funcs import json_to_df, make_df_float -from evdspy.EVDSlocal.requests_.real_requests import * -from evdspy.EVDSlocal.utils.github_actions import PytestTesting, GithubActions -from evdspy.EVDSlocal.index_requests.get_series_indexes_utils import ( - freq_enum, - Formulas, - AggregationType -) -# ........................................................................................... -from requests import HTTPError -import requests -from dataclasses import dataclass -from typing import Union, Callable -import pandas as pd -import json -from abc import ABC - - -# ....................................................................... ProxyManager -@dataclass -class ProxyManager: - proxy: Optional[str] = None - proxies: Optional[dict[Any, Any]] = None - - def get_proxies(self) -> Optional[dict[Any, Any]]: - if self.proxies is None: - if self.proxy is None: - proxies = None - else: - proxies = self.get_proxies_helper() - else: - proxies = self.proxies - return proxies - - def get_proxies_helper(self) -> Optional[dict[Any, Any]]: - if self.proxy is None: - return None - proxy = self.proxy - proxies = { - 'http': proxy, - 'https': proxy, - } - return proxies - - -# ....................................................................... Serialize -class Serialize(ABC): - """To check whether two Config Requests are perfect substitutes """ - - def serialize(self): - return json.dumps( - self, - default=lambda o: o.__dict__, - sort_keys=True, - indent=4) - - @property - def hash(self) -> str: - import hashlib - return str(int(hashlib.sha256(self.serialize().encode('utf-8')).hexdigest(), 16) % 10 ** 8) - - def __eq__(self, other): - return self.hash == other.hash - - -# ................................................................. UserRequestUtils -class UserRequestUtils: - @staticmethod - def looks_like_datagroup(string: Any): - if not isinstance(string, str): - return False - return 'bie_' in string - - @staticmethod - def clean_chars(string: str): - import re - if UserRequestUtils().looks_like_datagroup(string): - string = re.sub('[^0-9a-zA-Z._]+', '', string) - return string - string = string.replace("_", ".") - string = re.sub('[^0-9a-zA-Z.]+', '', string) - return string - - -def test_clean_chars(capsys): - with capsys.disabled(): - assert UserRequestUtils.clean_chars("AAA_BBB?*-()$? ") == "AAA.BBB" - assert UserRequestUtils.clean_chars("AAA..._BBB?*-()$? ") == "AAA....BBB" - assert UserRequestUtils.clean_chars("bie_BBB?*-()$? ") == "bie_BBB" - - -# ....................................................................... RequestConfig -# typings .......................................................... typings - -T_str_int_None = Union[str, int, None] -T_str_tuple_None = Union[str, tuple[str], None] -T_tuple_str_int_None = Union[str, int, tuple[str], tuple[int], None] -T_maybeDf = Union[pd.DataFrame, bool, None] - - -# ................................................................... RequestConfig -@dataclass -class RequestConfig(Serialize): - index: Union[str, tuple[str, ...], list[str, ...]] - start_date: str = default_start_date_fnc() - end_date: str = default_end_date_fnc() - frequency: T_str_int_None = None - formulas: T_tuple_str_int_None = None - aggregation: T_str_tuple_None = None - cache: bool = False - cache_name: str = "" - initial_index = None - # series_url_instance: SeriesUrlClass = SeriesUrlClass() - """ - Represents a user request for fetching data series from a specified source. - This class encapsulates the details necessary to construct and execute a data retrieval request, - handling various configurations like date ranges, frequency, and data aggregation types. It - also manages proxy settings and caching mechanisms to optimize data retrieval operations. - Attributes: - index (Union[str, Tuple[str]]): The identifier(s) for the data series to fetch. - start_date (str): The start date for the data retrieval in 'DD-MM-YYYY' format. - end_date (str): The end date for the data retrieval in 'DD-MM-YYYY' format. - frequency (Union[str, int, None]): The frequency at which data should be retrieved. - formulas (Union[str, int, Tuple[str, int], None]): The calculation types to apply to the data. - aggregation (Union[str, Tuple[str], None]): The aggregation methods to apply to the data. - cache (bool): Enables or disables caching of the request results. - cache_name (str): The name under which to store the cached results, if caching is enabled. - series_url_instance (SeriesUrlClass): An instance of SeriesUrlClass to handle URL creation. - Methods: - __post_init__: Initializes further attributes and performs checks after the basic initialization. - Raises: - ValueError: If the index is in an invalid format or necessary attributes are missing. - """ - - def check_type(self): - if UserRequestUtils.looks_like_datagroup(self.initial_index): - return "datagroup" - return "series" - - def __post_init__(self): - self.initial_index = self.index - self.correct_index() - self.correct_formulas_aggr() - self.check() - - def upper_index(self) -> None: - def upper(string: str) -> str: - if UserRequestUtils.looks_like_datagroup(self.initial_index): - return string - return string.upper() - - self.index = tuple(map(upper, self.index)) - - def clean(self) -> None: - self.index = tuple(map(UserRequestUtils.clean_chars, self.index)) - - def correct_index(self): - if "\n" in self.index or "\t" in self.index: - self.index = self.template_to_tuple(self.index) - self.index = tuple([self.index]) if not isinstance(self.index, (tuple, list,)) else self.index - self.clean() - self.upper_index() - self.check_index() - - def correct_formulas_aggr(self): - self.formulas = self.correct_type_to_tuple(self.formulas) - self.aggregation = self.correct_type_to_tuple(self.aggregation) - - def correct_type_to_tuple(self, value: any) -> Optional[tuple]: - if value is None: - return None - if isinstance(value, (list, tuple)): - if len(value) == len(self.index): - return value - if isinstance(value, (str,)): - return tuple(value for _ in self.index) - return tuple(value[0] for _ in self.index) - - @staticmethod - def template_to_tuple(index: str) -> tuple: - def clean(string: str): - return string.split("#")[0].strip() if len(string.split("#")) > 0 else None - - index = replace_all(index, "\t", "\n") - index_tuple = index.splitlines() - t = tuple(clean(x) for x in index_tuple) - return tuple(x for x in t if x is not None and len(x) > 3) - - def check_index(self) -> None: - if isinstance(self.index, (int, float,)): - raise ValueError("index must be a string ot tuple of string ") - - def freq_str(self) -> str: - if self.frequency is None: - return "" - if isinstance(self.frequency, int): - return f"&frequency={self.frequency}" - return freq_enum(self.frequency) - - def agr_form_type_to_str(self, value: Optional[tuple], part_name="aggregationTypes"): - if value is None: - return "" - value = tuple(map(str, value)) - string = "-".join(value) - return f"&{part_name}=" + string - - def aggregation_type_to_str(self) -> str: - self.aggregation = correct_types(self.aggregation, enum_class=AggregationType) - return self.agr_form_type_to_str(self.aggregation, "aggregationTypes") - - def formulas_to_str(self) -> str: - self.formulas = correct_types(self.formulas, enum_class=Formulas) - return self.agr_form_type_to_str(self.formulas, "formulas") - - def check(self) -> None: - if self.formulas is not None: - assert len(self.formulas) == len(self.index) - if self.aggregation is not None: - assert len(self.aggregation) == len(self.index) - if self.frequency is not None: - assert isinstance(self.frequency, (int, str,)) - - def create_series_part(self) -> str: - indexes = self.index - if isinstance(indexes, str): - indexes = tuple([indexes]) - return "-".join(indexes) - - -# ....................................................................... UrlSeries -class UrlSeries: - @property - def domain(self) -> str: - return "https://evds2.tcmb.gov.tr/service/evds" - - @property - def alias(self): - return "series=" - - -# ....................................................................... UrlDataGroup -class UrlDataGroup(UrlSeries): - @property - def alias(self): - return "datagroup=" - - -# ....................................................................... UrlBuilder -class UrlBuilder: - def __init__(self, - config: RequestConfig, - url_type=None) -> None: - self.config = config - self.series_part = self.config.create_series_part() - if not url_type: - self.get_url_type() - self.alias = self.url_type.alias - - def get_url_type(self): - url_type = UrlSeries() - if self.config.check_type() == "datagroup": - url_type = UrlDataGroup() - self.url_type = url_type - - def create_url_for_series(cls) -> str: - domain = cls.domain - return f"{domain}/{cls.alias}{cls.series_part}&startDate={cls.config.start_date}&endDate={cls.config.end_date}&type=json" - - @property - def domain(self) -> str: - return self.url_type.domain - - @property - def basic_url(self) -> str: - config = self.config - return f"{self.domain}/{self.alias}{self.series_part}&startDate={config.start_date}&endDate={config.end_date}&type=json" - - @property - def url(self) -> str: - config = self.config - if config.frequency is None and config.aggregation is None and config.formulas is None: - return self.basic_url - """ config parts """ - formulas_str = config.formulas_to_str() - aggregation_type_str = config.aggregation_type_to_str() - freq_string = config.freq_str() - """...""" - parts = ( - f"{self.domain}/{self.alias}{self.series_part}{freq_string}{formulas_str}{aggregation_type_str}", - f"startDate={config.start_date}", - f"endDate={config.end_date}", - "type=json" - ) - return "&".join(parts) - - -# .................................................................................. - -def create_cache_version(fnc: Callable): - @MyCache().cache - def fnc_cache(*args, **kw): - return fnc(*args, **kw) - - return fnc_cache - - -def cache_or_raw_fnc(fnc, cache=False): - if not cache: - return fnc - return create_cache_version(fnc) - - -def test_cache_or_raw_fnc(capsys): - with capsys.disabled(): - setup() - fnc = lambda x: x ** 2 - f = cache_or_raw_fnc(fnc, True) - f2 = cache_or_raw_fnc(fnc, False) - assert f(3) == 9 - assert f2(3) == 9 - assert id(f2) != id(f) - - -# ....................................................................... ApiRequester -class ApiRequester: - def __init__(self, url_builder: UrlBuilder, proxy_manager: ProxyManager): - self.url_builder = url_builder - self.proxy_manager = proxy_manager - self.proxies = self.proxy_manager.get_proxies() - self.url = self.url_builder.url - self.response = None - - def get(self): - return self.request() - - def __call__(self, *args, **kwargs): - return self.get() - - def dry_request(self) -> RequestConfig: - api_key = self.get_api_key(check=False) - print(f""" ---------------------------------- - [debug mode is turned on] - api_key = {self.obscure(api_key)} - proxies = {self.proxies} - url = {self.url} - ! request was not made because debug mode is turned on - in order to make the request run get_series(... , debug = False ) ---------------------------------- -""") - return self.url_builder.config - - def is_response_ok(self, response): - return isinstance(response, requests.Response) \ - and response.status_code == 200 - - @staticmethod - def obscure(string: str): - return ApikeyClass().obscure(string) - - - def get_api_key(self, check=True) -> str: - if PytestTesting().is_testing() or GithubActions().is_testing(): - api_key = self.get_api_key_while_testing() - else: - api_key = ApikeyClass().get_valid_api_key(check=check) - return api_key - - def get_api_key_while_testing(self): - return ApiClassWhileTesting()() - - def request(self) -> Any: - api_key = self.get_api_key() - if api_key is False: - if GithubActions().is_testing(): - return self.dry_request() - if PytestTesting().is_testing(): - raise NotImplementedError - proxies = self.proxy_manager.get_proxies() - - def local_request(url: str) -> requests.Response: - requester = RealRequestWithParam(url, - proxies=proxies, - api_key=api_key) - return requester.request() - - request_func = cache_or_raw_fnc(local_request, - cache=self.url_builder.config.cache) - response = False - try: - response = request_func(self.url) - except Exception as exc: - traceback.print_exc() - self.response = response - if not self.is_response_ok(response): - return False - # raise HTTPError(response=response) - return response.json() - - def is_ok(self) -> bool: - response = self.response - ok = isinstance(response, requests.Response) and response.status_code == 200 - if not ok: - self.dry_request() - raise ValueError("request Not Ok") - else: - print("") - return ok - - -import traceback - - -# ....................................................................... DataProcessor -class DataProcessor: - def __init__(self, data: Any): - self.data = data - - def process_to_dataframe(self) -> T_maybeDf: - if self.data is False: - return False - try: - df = json_to_df(self.data) - except Exception as e: - print(e) - traceback.print_exc() - return None - if isinstance(df, pd.DataFrame): - df = make_df_float(df) - return df - - def __call__(self, *args, **kwargs) -> T_maybeDf: - return self.process_to_dataframe() - - -def test_DataProcessor(capsys): - with capsys.disabled(): - d = DataProcessor(False) - print(d) diff --git a/evdspy/EVDSlocal/index_requests/user_requests/user_requests_2.py b/evdspy/EVDSlocal/index_requests/user_requests/user_requests_2.py deleted file mode 100644 index 409df77..0000000 --- a/evdspy/EVDSlocal/index_requests/user_requests/user_requests_2.py +++ /dev/null @@ -1,134 +0,0 @@ -""" -class UrlSeries: - @property - def domain(self) -> str: - return "https://evds2.tcmb.gov.tr/service/evds" - @property - def alias(self): - return "series=" - - - -""" -from evdspy.EVDSlocal.index_requests.user_requests import UrlSeries, UrlBuilder - - -class CategoriesMetadata(UrlSeries): - def get_url(self): - return f"{self.domain}/categories/type=json" - - @property - def url(self): - return self.get_url() - - -class DataGroup_info(): - def __init__(self, table_name): - self.table_name = table_name - - -class DatagroupsMetadata(UrlSeries): - def __init__(self, data_group: DataGroup_info): - self.data_group = data_group - - def get_url(self): - # "https://evds2.tcmb.gov.tr/service/evds/datagroups/mode=1&code=bie_yssk&type=json" - # self.table_name = self.table_name # "bie_yssk" - return f"{self.domain}/datagroups/mode=1&code={self.data_group.table_name}&type=json" - - @property - def url(self): - return self.get_url() - - -def test_CategoriesMetadata(capsys): - with capsys.disabled(): - d = DataGroup_info("bie_yssk") - assert d.table_name == "bie_yssk" - a = DatagroupsMetadata(d) - print(a.url) - b = CategoriesMetadata() - print(b.url) - - -class UrlBuilderMetadata(UrlBuilder): - ... - - def get_url(self): - ... - - def categories(self): - """""" - # 4.1 Category Service - # https://evds2.tcmb.gov.tr/service/evds/categories/type=json - - def datagroups_mode_1(self, name="bie_yssk"): - """ - https://evds2.tcmb.gov.tr/service/evds/datagroups/mode=1&code=bie_yssk&type=json - :return: - """ - ... - - def datagroups_mode_2(self, code=1): - """ - https://evds2.tcmb.gov.tr/service/evds/datagroups/mode=2&code=2&type=json - :return: - """ - ... - - def data_group_service(self): - """data_group_service""" - """ - The data group listing is based on the following filtering: - mode=0 Returns all data groups under all categories. - mode=1 Returns data group information according to a data group selection. - mode=2 Returns all data groups information according to a category - mode 1 - code=data group code - mode 2 - code=category code - https://evds2.tcmb.gov.tr/service/evds/datagroups/mode=1&code=bie_yssk&type=json - https://evds2.tcmb.gov.tr/service/evds/datagroups/mode=2&code=2&type=xml - DATAGROUP_CODE Data Group Code - DATAGROUP_NAME Data Group Name (Turkish) - DATAGROUP_NAME_ENG Data Group Name (English) - START_DATE Data Start Date - END_DATE Date End Date - FREQUENCY Original Frequency - FREQUENCY_STR Original Frequency Desc. - DATASOURCE Data Source (Turkish) - DATASOURCE_ENG Data Source (English) - METADATA_LINK Metadata Link (Turkish) - METADATA_LINK_ENG Metadata Link (English) - REV_POL_LINK Revision Policy Link (Turkish) - REV_POL_LINK_ENG Revision Policy Link (English) - APP_CHA_LINK Application Change Link (Turkish) - APP_CHA_LINK_ENG Application Change Link (English) - NOTE Information Note (Turkish) - NOTE_ENG Information Note (English) - """ - # 4.2 Data Group Service - - -from abc import ABC, abstractmethod - - -class GuideMetadata: - def get_meta(self): - ... - - def display_meta(self): - ... - - -class UrlBuilderMetadataCategories(UrlBuilderMetadata, GuideMetadata): - """ - UrlBuilderMetadataCategories - - """ - - -class UrlBuilderMetadataDataGroup(UrlBuilderMetadata, GuideMetadata): - """ - UrlBuilderMetadataCategories - """ diff --git a/evdspy/EVDSlocal/tests/test_user_requests.py b/evdspy/EVDSlocal/tests/test_user_requests.py index 834bd96..3f7a7fa 100644 --- a/evdspy/EVDSlocal/tests/test_user_requests.py +++ b/evdspy/EVDSlocal/tests/test_user_requests.py @@ -12,8 +12,9 @@ # from evdspy.EVDSlocal.index_requests.user_requests import from evdspy.EVDSlocal.config.apikey_class import ApikeyClass from evdspy.EVDSlocal.index_requests.get_series_indexes_utils import default_start_date_fnc, default_end_date_fnc -from evdspy.EVDSlocal.index_requests.user_requests.user_requests import ProxyManager, UrlBuilder, UrlSeries, \ - ApiRequester, DataProcessor, RequestConfig +from evdspy.EVDSlocal.index_requests.user_requests import (ProxyManager, UrlBuilder, UrlSeries, + ApiRequester, DataProcessor, RequestConfig) + # from evdspy.EVDSlocal.index_requests.get_series_indexes_utils import * try: @@ -22,8 +23,6 @@ pass - - @skip_if_gthub def test_get_api_key_file(capsys): with capsys.disabled(): @@ -49,6 +48,7 @@ def test_get_series_bridge(capsys): api_key=get_env_api_key(check=True)) assert is_df(df) + @skip_if_not_keyvalid def test_get_diff_series(capsys): with capsys.disabled(): @@ -71,6 +71,7 @@ def test_get_diff_series(capsys): api_key=get_env_api_key(check=True)) assert is_df(df) + # @skip_if_not_keyvalid def test_template_series(capsys): with capsys.disabled(): @@ -84,6 +85,7 @@ def test_template_series(capsys): print(a1.hash, a2.hash) assert a1 == a2 + @skip_if_not_keyvalid def test_a(capsys): with capsys.disabled(): @@ -114,6 +116,7 @@ def test_template_series2(capsys): print(a1.hash, a2.hash) assert a1 == a2 + # @skip_if_not_keyvalid def test_template_series3(capsys): @@ -179,6 +182,7 @@ def test_get_api_key_while_testing(capsys): a = ApiClassWhileTesting().key assert len(a) > 5 and 'lg' in a + @skip_if_not_keyvalid def test_get_series(capsys): with capsys.disabled(): @@ -236,6 +240,7 @@ def test_correct2(capsys): assert correct_types("avg", AggregationType) == "avg" assert correct_types(("avg", "min",), AggregationType) == ("avg", "min",) + @skip_if_not_keyvalid def test_mixedcase_get_series(capsys): index = """ @@ -255,6 +260,7 @@ def test_mixedcase_get_series(capsys): df = get_series(index) assert is_df(df) + @skip_if_not_keyvalid def test_gets_upper(capsys): with capsys.disabled(): @@ -276,6 +282,7 @@ def test_gets_upper(capsys): df = get_series(template) assert is_df(df) + # @skip_if_not_keyvalid def test_multi(capsys): template = """ @@ -291,6 +298,7 @@ def test_multi(capsys): assert all(map(lambda x: is_df(x), dfs)) tuple(map(lambda x: print(x.shape), dfs)) + @skip_if_not_keyvalid def test_get_series_b(capsys): index = """ diff --git a/evdspy/__init__.py b/evdspy/__init__.py index c0c8f88..19d0710 100644 --- a/evdspy/__init__.py +++ b/evdspy/__init__.py @@ -6,8 +6,15 @@ # ------------------------------------------------------ from evdspy.EVDSlocal.main_ import * from evdspy.EVDSlocal.index_requests.get_series_indexes import ( - default_start_date_fnc, - default_end_date_fnc, + # default_start_date_fnc, + # default_end_date_fnc, get_series, ) -from evdspy.EVDSlocal.utils.utils_general import ls \ No newline at end of file +# from evdspy.EVDSlocal.index_requests.user_requests.User_request_utils import +from evdspy.EVDSlocal.utils.utils_general import ls + +from evdspy.EVDSlocal.index_requests.get_series_indexes_utils import ( + default_start_date_fnc, + default_end_date_fnc, + correct_types, +) \ No newline at end of file From 15db01a1792de1e471d64284cd8ed5847bfcaa49 Mon Sep 17 00:00:00 2001 From: Sermet Pekin Date: Tue, 23 Apr 2024 00:20:49 +0300 Subject: [PATCH 2/4] . --- .../index_requests/user_requests/__init__.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/evdspy/EVDSlocal/index_requests/user_requests/__init__.py b/evdspy/EVDSlocal/index_requests/user_requests/__init__.py index c290875..8d274cc 100644 --- a/evdspy/EVDSlocal/index_requests/user_requests/__init__.py +++ b/evdspy/EVDSlocal/index_requests/user_requests/__init__.py @@ -1,13 +1,7 @@ -# from evdspy.EVDSlocal.index_requests.user_requests.Proxy_manager import ProxyManager, UrlBuilder, UrlSeries, \ -# ApiRequester, DataProcessor, RequestConfig - -from .Url_builder import * -from .Data_processor import * -from .Data_processor import * -from .Data_processor import * -from .Proxy_manager import * -from .Serialize import * -from .Url_series import * +from .Url_builder import UrlBuilder +from .Data_processor import DataProcessor +from .Proxy_manager import ProxyManager +from .Serialize import Serialize +from .Url_series import UrlSeries, UrlDataGroup from .User_req_typings import * -from .user_requests import * -from .Api_requester import ApiRequester \ No newline at end of file +from .Api_requester import ApiRequester From ec236d64c4919353adf257a5af08e42863b00ed2 Mon Sep 17 00:00:00 2001 From: Sermet Pekin Date: Tue, 23 Apr 2024 00:24:39 +0300 Subject: [PATCH 3/4] . --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 2b88a3b..f5fa474 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ # Byte-compiled / optimized / DLL files __pycache__/ +!Api_requester.py + + *.py[cod] *$py.class .idea/ From 32a399f455474e91e0760eaf80dd7bf983fbbeba Mon Sep 17 00:00:00 2001 From: Sermet Pekin Date: Tue, 23 Apr 2024 00:25:50 +0300 Subject: [PATCH 4/4] . --- .gitignore | 4 +- .../user_requests/Api_requester.py | 96 +++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 evdspy/EVDSlocal/index_requests/user_requests/Api_requester.py diff --git a/.gitignore b/.gitignore index f5fa474..1bb52b6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ # Byte-compiled / optimized / DLL files __pycache__/ -!Api_requester.py *.py[cod] @@ -271,3 +270,6 @@ cython_debug/ #.idea/ /_version.py /EVDSlocal/APIKEY_FOLDER/ + + +!Api_requester.py diff --git a/evdspy/EVDSlocal/index_requests/user_requests/Api_requester.py b/evdspy/EVDSlocal/index_requests/user_requests/Api_requester.py new file mode 100644 index 0000000..b44bce0 --- /dev/null +++ b/evdspy/EVDSlocal/index_requests/user_requests/Api_requester.py @@ -0,0 +1,96 @@ +# ........................................................................................... +import traceback +import requests +# ........................................................................................... +from evdspy.EVDSlocal.index_requests.user_requests import UrlBuilder, ProxyManager, RequestConfig +from evdspy.EVDSlocal.index_requests.user_requests.User_request_utils import cache_or_raw_fnc +from evdspy.EVDSlocal.utils.utils_test import ApiClassWhileTesting +from evdspy.EVDSlocal.requests_.real_requests import * +from evdspy.EVDSlocal.utils.github_actions import PytestTesting, GithubActions +# ........................................................................................... + + +# ....................................................................... ApiRequester +class ApiRequester: + def __init__(self, url_builder: UrlBuilder, proxy_manager: ProxyManager): + self.url_builder = url_builder + self.proxy_manager = proxy_manager + self.proxies = self.proxy_manager.get_proxies() + self.url = self.url_builder.url + self.response = None + + def get(self): + return self.request() + + def __call__(self, *args, **kwargs): + return self.get() + + def dry_request(self) -> RequestConfig: + api_key = self.get_api_key(check=False) + print(f""" +--------------------------------- + [debug mode is turned on] + api_key = {self.obscure(api_key)} + proxies = {self.proxies} + url = {self.url} + ! request was not made because debug mode is turned on + in order to make the request run get_series(... , debug = False ) +--------------------------------- +""") + return self.url_builder.config + + def is_response_ok(self, response): + return isinstance(response, requests.Response) \ + and response.status_code == 200 + + @staticmethod + def obscure(string: str): + return ApikeyClass().obscure(string) + + def get_api_key(self, check=True) -> str: + if PytestTesting().is_testing() or GithubActions().is_testing(): + api_key = self.get_api_key_while_testing() + else: + api_key = ApikeyClass().get_valid_api_key(check=check) + return api_key + + def get_api_key_while_testing(self): + return ApiClassWhileTesting()() + + def request(self) -> Any: + api_key = self.get_api_key() + if api_key is False: + if GithubActions().is_testing(): + return self.dry_request() + if PytestTesting().is_testing(): + raise NotImplementedError + proxies = self.proxy_manager.get_proxies() + + def local_request(url: str) -> requests.Response: + requester = RealRequestWithParam(url, + proxies=proxies, + api_key=api_key) + return requester.request() + + request_func = cache_or_raw_fnc(local_request, + cache=self.url_builder.config.cache) + response = False + try: + response = request_func(self.url) + except Exception as exc: + traceback.print_exc() + self.response = response + if not self.is_response_ok(response): + return False + # raise HTTPError(response=response) + return response.json() + + def is_ok(self) -> bool: + response = self.response + ok = isinstance(response, requests.Response) and response.status_code == 200 + if not ok: + self.dry_request() + raise ValueError("request Not Ok") + else: + print("") + return ok