diff --git a/Python/AnP/Abstracts/BrowserAbstract.py b/Python/AnP/Abstracts/BrowserAbstract.py index 994507c..bc87e3e 100644 --- a/Python/AnP/Abstracts/BrowserAbstract.py +++ b/Python/AnP/Abstracts/BrowserAbstract.py @@ -10,6 +10,7 @@ from AnP.DSLs.ScrapDSL import ( If, While, DoWhile, Download, Selector, Condition ) from AnP.Utils.Common import Common +from AnP.Utils.Checks import Check from AnP.Abstracts.ModelAbstract import ModelAbstract from AnP.Models.BrowserTabsModel import BrowserBoxModel @@ -251,12 +252,15 @@ class BrowserAbstract(ModelAbstract): elif isinstance(action, bool): return action elif isinstance(action, Selector): - return self.get_selector(action, box) + return not not self.get_selector(action, box) elif callable(action): try: return action(shared) except Exception as exception: - action.exception = exception + self.anp.exception(exception, "anp_browser_condition_exception", { + "key" : self.key + }) + # action.exception = exception else: action.print_errors and self.anp.print("warning", "anp_browser_condition_not_supported", { "key" : self.key @@ -290,22 +294,28 @@ class BrowserAbstract(ModelAbstract): def _if(self:Self, action:If, shared:dict[str, Any|None], box:BrowserBoxModel) -> None: - if not isinstance(action.condition, Condition): - action.condition = Condition((action.condition,)) + # if not isinstance(action.condition, Condition): + # action.condition = Condition((action.condition,)) - self.do(action.condition, shared, box) + # self.do(action.condition, shared, box) - if action.condition.done and action.condition.exception is None: + # if action.condition.done and action.condition.exception is None: - subactions:Do = Do(*((action.if_actions if action.condition.ok else action.else_actions) or [])) + # subactions:Do = Do(*((action.if_actions if action.condition.ok else action.else_actions) or [])) - self.do(subactions, shared, box) - action.done = subactions.done and subactions.exception is None - action.exception = subactions.exception + # self.do(subactions, shared, box) + # action.done = subactions.done and subactions.exception is None + # action.exception = subactions.exception - else: - action.done = action.condition.done - action.exception = action.condition.exception + # else: + # action.done = action.condition.done + # action.exception = action.condition.exception + + subactions:Do = Do(*((action.if_actions if self.__condition(action.condition, shared, box) else action.else_actions) or [])) + + self.do(subactions, shared, box) + action.done = subactions.done and subactions.exception is None + action.exception = subactions.exception def _while(self:Self, action:While, shared:dict[str, Any|None], box:BrowserBoxModel) -> None: @@ -319,29 +329,39 @@ class BrowserAbstract(ModelAbstract): self.do(action.condition, shared, box) - if not action.condition.done or action.condition.exception is not None: - action.done = action.condition.done - action.exception = action.condition.exception - break - - if action.condition.ok: - - subactions:Do = Do(*action.actions) - - self.do(subactions, shared, box) - if not subactions.done or subactions.exception is not None: - action.exception = subactions.exception - action.done = False - break - - else: + # if not action.condition.done or action.condition.exception is not None: + if not self.__condition(action.condition, shared, box): action.done = True + # action.done = action.condition.done + # action.exception = action.condition.exception break + subactions:Do = Do(*action.actions) + + self.do(subactions, shared, box) + if not subactions.done or subactions.exception is not None: + action.exception = subactions.exception + action.done = False + break + + # if action.condition.ok: + + # subactions:Do = Do(*action.actions) + + # self.do(subactions, shared, box) + # if not subactions.done or subactions.exception is not None: + # action.exception = subactions.exception + # action.done = False + # break + + # else: + # action.done = True + # break + def do_while(self:Self, action:DoWhile, shared:dict[str, Any|None], box:BrowserBoxModel) -> None: - if not isinstance(action.condition, Condition): - action.condition = Condition((action.condition,)) + # if not isinstance(action.condition, Condition): + # action.condition = Condition((action.condition,)) while True: if not self.is_working(box): @@ -359,12 +379,16 @@ class BrowserAbstract(ModelAbstract): subactions = Do(*action.actions) self.do(subactions, shared, box) - if not action.condition.done or action.condition.exception is not None: - action.done = action.condition.done - action.exception = action.condition.exception - break + # if not action.condition.done or action.condition.exception is not None: + # action.done = action.condition.done + # action.exception = action.condition.exception + # break - if not action.condition.ok: + # if not action.condition.ok: + # action.done = True + # break + + if not self.__condition(action.condition, shared, box): action.done = True break diff --git a/Python/AnP/DSLs/ScrapDSL.py b/Python/AnP/DSLs/ScrapDSL.py index 6dd695b..8c36ca0 100644 --- a/Python/AnP/DSLs/ScrapDSL.py +++ b/Python/AnP/DSLs/ScrapDSL.py @@ -17,7 +17,10 @@ class Action: class Execute(Action): def __init__(self:Self, callback:Callable[[dict[str, Any|None]], None]) -> None: super().__init__() + self.callback:Callable[[dict[str, Any|None]], None] = callback + + self.wait = False class Open(Action): def __init__(self:Self, key:str, url:Optional[str] = None) -> None: @@ -93,6 +96,8 @@ class Get(Action): self.pattern:REPattern|None = None self.matches:str|Callable[[REMatch], str]|None = None + self.wait = False + if pattern is not None: self.pattern, self.matches = pattern @@ -103,16 +108,22 @@ class ForEach(Action): callback:Optional[Callable[[Any|None], None]] = None ) -> None: super().__init__() + self.items:Selector|Sequence[Any|None] = items self.actions:Sequence[Action] = actions self.callback:Callable[[Any|None], None]|None = callback + self.wait = False + class Condition(Action): def __init__(self:Self, conditions:Sequence[Callable[[dict[str, Any|None]], bool]]|Selector|Self) -> None: super().__init__() + self.conditions:Sequence[Callable[[dict[str, Any|None]], bool]|Selector|Condition|bool] = conditions self.ok:bool = False + self.wait = False + class And(Condition):pass class Or(Condition):pass @@ -130,28 +141,37 @@ class If(Action): else_actions:Optional[Sequence[Action]] = None ) -> None: super().__init__() + self.condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool = condition self.if_actions:Sequence[Action]|None = if_actions self.else_actions:Sequence[Action]|None = else_actions + self.wait = False + class While(Action): def __init__(self:Self, condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool, actions:Sequence[Action] ) -> None: super().__init__() + self.condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool = condition self.actions:Sequence[Action] = actions + self.wait = False + class DoWhile(Action): def __init__(self:Self, condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool, actions:Sequence[Action] ) -> None: super().__init__() + self.condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool = condition self.actions:Sequence[Action] = actions + self.wait = False + class Download(Action): def __init__(self:Self, url:str, diff --git a/Python/AnP/Drivers/SQLiteDriver.py b/Python/AnP/Drivers/SQLiteDriver.py index e99dadf..a400b2e 100644 --- a/Python/AnP/Drivers/SQLiteDriver.py +++ b/Python/AnP/Drivers/SQLiteDriver.py @@ -61,7 +61,7 @@ class SQLiteDriver(DatabaseAbstract): return ( "null" if value is None else - "'" + value + "'" if Check.is_string(value) else + "'" + value.replace("'", "''") + "'" if Check.is_string(value) else ("1" if value is True else "0") if Check.is_boolean(value) else # "'" + value.strftime("%Y-%m-%d %H:%M:%S") + "'" if Check.is_date(value) else "'" + value.isoformat(sep = " ") + "'" if Check.is_date(value) else @@ -83,6 +83,8 @@ class SQLiteDriver(DatabaseAbstract): position:int|None subquery:str = "" + query = RE.SQLITE_COMMENTS.sub(r'', query) + while True: lower:str|None = None @@ -142,25 +144,39 @@ class SQLiteDriver(DatabaseAbstract): for i, sentence in enumerate(sentences): - row:list[Any] - columns:list[str] + try: - cursor.execute(sentence) - columns = [ - column[0] for column in cursor.description - ] if cursor.description is not None else [] - self.__connection.commit() - - for row in cursor.fetchall(): - response.add(i, dict(zip(columns, row))) + row:list[Any] + columns:list[str] + table:list[Any|None] + + cursor.execute(sentence) + columns = [ + column[0] for column in cursor.description + ] if cursor.description is not None else [] + self.__connection.commit() + table = cursor.fetchall() + + if len(table): + for row in table: + response.add(i, dict(zip(columns, row))) + else: + response.tables.append([]) + + except Exception as exception: + self.anp.exception(exception, "anp_sqlite_driver_execution_sentence_exception", { + "path" : self.__path, + "i" : i, + "sentence" : sentence + }) + print(sentence) + break self.__connection.commit() except Exception as exception: self.anp.exception(exception, "anp_sqlite_driver_execution_exception", { - "path" : self.__path, - "i" : i, - "sentence" : sentence + "path" : self.__path }) self.__connection.rollback() diff --git a/Python/AnP/Drivers/XMLScrapDriver.py b/Python/AnP/Drivers/XMLScrapDriver.py index 3275030..d655ff4 100644 --- a/Python/AnP/Drivers/XMLScrapDriver.py +++ b/Python/AnP/Drivers/XMLScrapDriver.py @@ -47,7 +47,7 @@ class XMLScrapDriver(BrowserAbstract): self.__load(action, action.key, action.url, shared, box) def go(self:Self, action:Go, shared:dict[str, Any|None], box:BrowserBoxModel) -> None: - self.__load(action, action.key, action.url, shared, box) + self.__load(action, None, action.url, shared, box) def get_selector(self:Self, selector:Selector, box:BrowserBoxModel, force_multiple:bool = False) -> Any|list[Any]|None: diff --git a/Python/AnP/Managers/ControllersManager.py b/Python/AnP/Managers/ControllersManager.py index 929ea73..77f7d46 100644 --- a/Python/AnP/Managers/ControllersManager.py +++ b/Python/AnP/Managers/ControllersManager.py @@ -62,11 +62,10 @@ class ControllersManager: controller if Check.is_dictionary(controller) else None)) - def execute(self:Self, key:str, method:str, request:RequestModel) -> bool: + def execute(self:Self, key:str, method:str, request:RequestModel) -> Any|None: if key in self.__controllers and hasattr(self.__controllers[key], method): - getattr(self.__controllers[key], method)(request) - return True - return False + return getattr(self.__controllers[key], method)(request) + return None def get(self:Self, key:str, Type:type[ControllerType]) -> ControllerType|None: if key in self.__controllers and isinstance(self.__controllers[key], Type): diff --git a/Python/AnP/Managers/DispatchesManager.py b/Python/AnP/Managers/DispatchesManager.py index c413755..e197a43 100644 --- a/Python/AnP/Managers/DispatchesManager.py +++ b/Python/AnP/Managers/DispatchesManager.py @@ -35,17 +35,17 @@ class DispatchesManager: for subinputs in self.anp.files.load_json(inputs, True): - key:str + via:str subinputs:dict[str, Any|None] - for key, subinputs in subinputs.items(): - if Check.is_mark_key(key) and dispatch is None: + for via, subinputs in subinputs.items(): + if Check.is_mark_key(via) and dispatch is None: continue - via:str + key:str dispatch:Any|None - for via, dispatch in subinputs.items(): + for key, dispatch in subinputs.items(): DispatchClass:type[DispatchAbstract]|None diff --git a/Python/AnP/Models/HashModel.py b/Python/AnP/Models/HashModel.py index 162d24e..5074cd3 100644 --- a/Python/AnP/Models/HashModel.py +++ b/Python/AnP/Models/HashModel.py @@ -80,4 +80,7 @@ class HashModel(): return done def compare(self:Self, other:Self) -> bool: - return all(getattr(self, name) == getattr(other, name) for name in self.__hashes) \ No newline at end of file + return all(getattr(self, name) == getattr(other, name) for name in self.__hashes) + + def get_dictionary(self:Self) -> dict[str, str]: + return {name : getattr(self, name) for name in self.__hashes} \ No newline at end of file diff --git a/Python/AnP/Utils/Patterns.py b/Python/AnP/Utils/Patterns.py index 1210d44..67b659b 100644 --- a/Python/AnP/Utils/Patterns.py +++ b/Python/AnP/Utils/Patterns.py @@ -21,6 +21,7 @@ class RE: HTTP_REQUEST:REPattern = re_compile(r'^([^\s]+)\s([^\s\?\#]+)(?:\?([^#]+))?(?:\#([^\s]+))?\s([^\/]+)\/([0-9\.]+)$') PYTHON_MODULES_PATH:REPattern = re_compile(r'[\/\\]python[0-9\.]+[\/\\]site-packages[\/\\]?$', RE_IGNORECASE) ODBC_STRING_VARIABLE:REPattern = re_compile(r'\{([a-z_][a-z0-9_]*)\}|@([a-z0-9_]+)', RE_IGNORECASE) + SQLITE_COMMENTS:REPattern = re_compile(r'--[^\r\n]*|\/\*(?:[^\*]|\*+[^\/\*]|[\r\n]+)*\*+\/') # ORM ORM_MAIN_BLOCKS:REPattern = re_compile(r'((?:(?!(?:(?:\r\n){2}|[\r\n]{2}))(?:.|[\r\n]))+)(?:(?:\r\n){2}|[\r\n]{2})((?:.+|[\r\n]+)+)')