KyMAN/AnP
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(py): Fixed Scrap conditions system.

Browse Source
This commit is contained in:
KyMAN 2026-07-17 07:30:27 +02:00
parent dc026f162e
commit f1807f5f9d
8 changed files with 124 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
Python/AnP/Abstracts/BrowserAbstract.py
View File

@ -10,6 +10,7 @@ from AnP.DSLs.ScrapDSL import (
If, While, DoWhile, Download, Selector, Condition If, While, DoWhile, Download, Selector, Condition
) )
from AnP.Utils.Common import Common from AnP.Utils.Common import Common
from AnP.Utils.Checks import Check
from AnP.Abstracts.ModelAbstract import ModelAbstract from AnP.Abstracts.ModelAbstract import ModelAbstract
from AnP.Models.BrowserTabsModel import BrowserBoxModel from AnP.Models.BrowserTabsModel import BrowserBoxModel
@ -251,12 +252,15 @@ class BrowserAbstract(ModelAbstract):
elif isinstance(action, bool): elif isinstance(action, bool):
return action return action
elif isinstance(action, Selector): elif isinstance(action, Selector):
return self.get_selector(action, box) return not not self.get_selector(action, box)
elif callable(action): elif callable(action):
try: try:
return action(shared) return action(shared)
except Exception as exception: except Exception as exception:
action.exception = exception self.anp.exception(exception, "anp_browser_condition_exception", {
"key" : self.key
})
# action.exception = exception
else: else:
action.print_errors and self.anp.print("warning", "anp_browser_condition_not_supported", { action.print_errors and self.anp.print("warning", "anp_browser_condition_not_supported", {
"key" : self.key "key" : self.key
@ -290,22 +294,28 @@ class BrowserAbstract(ModelAbstract):
def _if(self:Self, action:If, shared:dict[str, Any|None], box:BrowserBoxModel) -> None: def _if(self:Self, action:If, shared:dict[str, Any|None], box:BrowserBoxModel) -> None:
if not isinstance(action.condition, Condition): # if not isinstance(action.condition, Condition):
action.condition = Condition((action.condition,)) # action.condition = Condition((action.condition,))
self.do(action.condition, shared, box) # self.do(action.condition, shared, box)
if action.condition.done and action.condition.exception is None: # if action.condition.done and action.condition.exception is None:
subactions:Do = Do(*((action.if_actions if action.condition.ok else action.else_actions) or [])) # subactions:Do = Do(*((action.if_actions if action.condition.ok else action.else_actions) or []))
self.do(subactions, shared, box) # self.do(subactions, shared, box)
action.done = subactions.done and subactions.exception is None # action.done = subactions.done and subactions.exception is None
action.exception = subactions.exception # action.exception = subactions.exception
else: # else:
action.done = action.condition.done # action.done = action.condition.done
action.exception = action.condition.exception # action.exception = action.condition.exception
subactions:Do = Do(*((action.if_actions if self.__condition(action.condition, shared, box) else action.else_actions) or []))
self.do(subactions, shared, box)
action.done = subactions.done and subactions.exception is None
action.exception = subactions.exception
def _while(self:Self, action:While, shared:dict[str, Any|None], box:BrowserBoxModel) -> None: def _while(self:Self, action:While, shared:dict[str, Any|None], box:BrowserBoxModel) -> None:
@ -319,29 +329,39 @@ class BrowserAbstract(ModelAbstract):
self.do(action.condition, shared, box) self.do(action.condition, shared, box)
if not action.condition.done or action.condition.exception is not None: # if not action.condition.done or action.condition.exception is not None:
action.done = action.condition.done if not self.__condition(action.condition, shared, box):
action.exception = action.condition.exception
break
if action.condition.ok:
subactions:Do = Do(*action.actions)
self.do(subactions, shared, box)
if not subactions.done or subactions.exception is not None:
action.exception = subactions.exception
action.done = False
break
else:
action.done = True action.done = True
# action.done = action.condition.done
# action.exception = action.condition.exception
break break
subactions:Do = Do(*action.actions)
self.do(subactions, shared, box)
if not subactions.done or subactions.exception is not None:
action.exception = subactions.exception
action.done = False
break
# if action.condition.ok:
# subactions:Do = Do(*action.actions)
# self.do(subactions, shared, box)
# if not subactions.done or subactions.exception is not None:
# action.exception = subactions.exception
# action.done = False
# break
# else:
# action.done = True
# break
def do_while(self:Self, action:DoWhile, shared:dict[str, Any|None], box:BrowserBoxModel) -> None: def do_while(self:Self, action:DoWhile, shared:dict[str, Any|None], box:BrowserBoxModel) -> None:
if not isinstance(action.condition, Condition): # if not isinstance(action.condition, Condition):
action.condition = Condition((action.condition,)) # action.condition = Condition((action.condition,))
while True: while True:
if not self.is_working(box): if not self.is_working(box):
@ -359,12 +379,16 @@ class BrowserAbstract(ModelAbstract):
subactions = Do(*action.actions) subactions = Do(*action.actions)
self.do(subactions, shared, box) self.do(subactions, shared, box)
if not action.condition.done or action.condition.exception is not None: # if not action.condition.done or action.condition.exception is not None:
action.done = action.condition.done # action.done = action.condition.done
action.exception = action.condition.exception # action.exception = action.condition.exception
break # break
if not action.condition.ok: # if not action.condition.ok:
# action.done = True
# break
if not self.__condition(action.condition, shared, box):
action.done = True action.done = True
break break

20
Python/AnP/DSLs/ScrapDSL.py
View File

@ -17,7 +17,10 @@ class Action:
class Execute(Action): class Execute(Action):
def __init__(self:Self, callback:Callable[[dict[str, Any|None]], None]) -> None: def __init__(self:Self, callback:Callable[[dict[str, Any|None]], None]) -> None:
super().__init__() super().__init__()
self.callback:Callable[[dict[str, Any|None]], None] = callback self.callback:Callable[[dict[str, Any|None]], None] = callback
self.wait = False
class Open(Action): class Open(Action):
def __init__(self:Self, key:str, url:Optional[str] = None) -> None: def __init__(self:Self, key:str, url:Optional[str] = None) -> None:
@ -93,6 +96,8 @@ class Get(Action):
self.pattern:REPattern|None = None self.pattern:REPattern|None = None
self.matches:str|Callable[[REMatch], str]|None = None self.matches:str|Callable[[REMatch], str]|None = None
self.wait = False
if pattern is not None: if pattern is not None:
self.pattern, self.matches = pattern self.pattern, self.matches = pattern
@ -103,16 +108,22 @@ class ForEach(Action):
callback:Optional[Callable[[Any|None], None]] = None callback:Optional[Callable[[Any|None], None]] = None
) -> None: ) -> None:
super().__init__() super().__init__()
self.items:Selector|Sequence[Any|None] = items self.items:Selector|Sequence[Any|None] = items
self.actions:Sequence[Action] = actions self.actions:Sequence[Action] = actions
self.callback:Callable[[Any|None], None]|None = callback self.callback:Callable[[Any|None], None]|None = callback
self.wait = False
class Condition(Action): class Condition(Action):
def __init__(self:Self, conditions:Sequence[Callable[[dict[str, Any|None]], bool]]|Selector|Self) -> None: def __init__(self:Self, conditions:Sequence[Callable[[dict[str, Any|None]], bool]]|Selector|Self) -> None:
super().__init__() super().__init__()
self.conditions:Sequence[Callable[[dict[str, Any|None]], bool]|Selector|Condition|bool] = conditions self.conditions:Sequence[Callable[[dict[str, Any|None]], bool]|Selector|Condition|bool] = conditions
self.ok:bool = False self.ok:bool = False
self.wait = False
class And(Condition):pass class And(Condition):pass
class Or(Condition):pass class Or(Condition):pass
@ -130,28 +141,37 @@ class If(Action):
else_actions:Optional[Sequence[Action]] = None else_actions:Optional[Sequence[Action]] = None
) -> None: ) -> None:
super().__init__() super().__init__()
self.condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool = condition self.condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool = condition
self.if_actions:Sequence[Action]|None = if_actions self.if_actions:Sequence[Action]|None = if_actions
self.else_actions:Sequence[Action]|None = else_actions self.else_actions:Sequence[Action]|None = else_actions
self.wait = False
class While(Action): class While(Action):
def __init__(self:Self, def __init__(self:Self,
condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool, condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool,
actions:Sequence[Action] actions:Sequence[Action]
) -> None: ) -> None:
super().__init__() super().__init__()
self.condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool = condition self.condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool = condition
self.actions:Sequence[Action] = actions self.actions:Sequence[Action] = actions
self.wait = False
class DoWhile(Action): class DoWhile(Action):
def __init__(self:Self, def __init__(self:Self,
condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool, condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool,
actions:Sequence[Action] actions:Sequence[Action]
) -> None: ) -> None:
super().__init__() super().__init__()
self.condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool = condition self.condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool = condition
self.actions:Sequence[Action] = actions self.actions:Sequence[Action] = actions
self.wait = False
class Download(Action): class Download(Action):
def __init__(self:Self, def __init__(self:Self,
url:str, url:str,

44
Python/AnP/Drivers/SQLiteDriver.py
View File

@ -61,7 +61,7 @@ class SQLiteDriver(DatabaseAbstract):
return ( return (
"null" if value is None else "null" if value is None else
"'" + value + "'" if Check.is_string(value) else "'" + value.replace("'", "''") + "'" if Check.is_string(value) else
("1" if value is True else "0") if Check.is_boolean(value) else ("1" if value is True else "0") if Check.is_boolean(value) else
# "'" + value.strftime("%Y-%m-%d %H:%M:%S") + "'" if Check.is_date(value) else # "'" + value.strftime("%Y-%m-%d %H:%M:%S") + "'" if Check.is_date(value) else
"'" + value.isoformat(sep = " ") + "'" if Check.is_date(value) else "'" + value.isoformat(sep = " ") + "'" if Check.is_date(value) else
@ -83,6 +83,8 @@ class SQLiteDriver(DatabaseAbstract):
position:int|None position:int|None
subquery:str = "" subquery:str = ""
query = RE.SQLITE_COMMENTS.sub(r'', query)
while True: while True:
lower:str|None = None lower:str|None = None
@ -142,25 +144,39 @@ class SQLiteDriver(DatabaseAbstract):
for i, sentence in enumerate(sentences): for i, sentence in enumerate(sentences):
row:list[Any] try:
columns:list[str]
cursor.execute(sentence) row:list[Any]
columns = [ columns:list[str]
column[0] for column in cursor.description table:list[Any|None]
] if cursor.description is not None else []
self.__connection.commit() cursor.execute(sentence)
columns = [
for row in cursor.fetchall(): column[0] for column in cursor.description
response.add(i, dict(zip(columns, row))) ] if cursor.description is not None else []
self.__connection.commit()
table = cursor.fetchall()
if len(table):
for row in table:
response.add(i, dict(zip(columns, row)))
else:
response.tables.append([])
except Exception as exception:
self.anp.exception(exception, "anp_sqlite_driver_execution_sentence_exception", {
"path" : self.__path,
"i" : i,
"sentence" : sentence
})
print(sentence)
break
self.__connection.commit() self.__connection.commit()
except Exception as exception: except Exception as exception:
self.anp.exception(exception, "anp_sqlite_driver_execution_exception", { self.anp.exception(exception, "anp_sqlite_driver_execution_exception", {
"path" : self.__path, "path" : self.__path
"i" : i,
"sentence" : sentence
}) })
self.__connection.rollback() self.__connection.rollback()

2
Python/AnP/Drivers/XMLScrapDriver.py
View File

@ -47,7 +47,7 @@ class XMLScrapDriver(BrowserAbstract):
self.__load(action, action.key, action.url, shared, box) self.__load(action, action.key, action.url, shared, box)
def go(self:Self, action:Go, shared:dict[str, Any|None], box:BrowserBoxModel) -> None: def go(self:Self, action:Go, shared:dict[str, Any|None], box:BrowserBoxModel) -> None:
self.__load(action, action.key, action.url, shared, box) self.__load(action, None, action.url, shared, box)
def get_selector(self:Self, selector:Selector, box:BrowserBoxModel, force_multiple:bool = False) -> Any|list[Any]|None: def get_selector(self:Self, selector:Selector, box:BrowserBoxModel, force_multiple:bool = False) -> Any|list[Any]|None:

7
Python/AnP/Managers/ControllersManager.py
View File

@ -62,11 +62,10 @@ class ControllersManager:
controller if Check.is_dictionary(controller) else controller if Check.is_dictionary(controller) else
None)) None))
def execute(self:Self, key:str, method:str, request:RequestModel) -> bool: def execute(self:Self, key:str, method:str, request:RequestModel) -> Any|None:
if key in self.__controllers and hasattr(self.__controllers[key], method): if key in self.__controllers and hasattr(self.__controllers[key], method):
getattr(self.__controllers[key], method)(request) return getattr(self.__controllers[key], method)(request)
return True return None
return False
def get(self:Self, key:str, Type:type[ControllerType]) -> ControllerType|None: def get(self:Self, key:str, Type:type[ControllerType]) -> ControllerType|None:
if key in self.__controllers and isinstance(self.__controllers[key], Type): if key in self.__controllers and isinstance(self.__controllers[key], Type):

10
Python/AnP/Managers/DispatchesManager.py
View File

@ -35,17 +35,17 @@ class DispatchesManager:
for subinputs in self.anp.files.load_json(inputs, True): for subinputs in self.anp.files.load_json(inputs, True):
key:str via:str
subinputs:dict[str, Any|None] subinputs:dict[str, Any|None]
for key, subinputs in subinputs.items(): for via, subinputs in subinputs.items():
if Check.is_mark_key(key) and dispatch is None: if Check.is_mark_key(via) and dispatch is None:
continue continue
via:str key:str
dispatch:Any|None dispatch:Any|None
for via, dispatch in subinputs.items(): for key, dispatch in subinputs.items():
DispatchClass:type[DispatchAbstract]|None DispatchClass:type[DispatchAbstract]|None

5
Python/AnP/Models/HashModel.py
View File

@ -80,4 +80,7 @@ class HashModel():
return done return done
def compare(self:Self, other:Self) -> bool: def compare(self:Self, other:Self) -> bool:
return all(getattr(self, name) == getattr(other, name) for name in self.__hashes) return all(getattr(self, name) == getattr(other, name) for name in self.__hashes)
def get_dictionary(self:Self) -> dict[str, str]:
return {name : getattr(self, name) for name in self.__hashes}

1
Python/AnP/Utils/Patterns.py
View File

@ -21,6 +21,7 @@ class RE:
HTTP_REQUEST:REPattern = re_compile(r'^([^\s]+)\s([^\s\?\#]+)(?:\?([^#]+))?(?:\#([^\s]+))?\s([^\/]+)\/([0-9\.]+)$') HTTP_REQUEST:REPattern = re_compile(r'^([^\s]+)\s([^\s\?\#]+)(?:\?([^#]+))?(?:\#([^\s]+))?\s([^\/]+)\/([0-9\.]+)$')
PYTHON_MODULES_PATH:REPattern = re_compile(r'[\/\\]python[0-9\.]+[\/\\]site-packages[\/\\]?$', RE_IGNORECASE) PYTHON_MODULES_PATH:REPattern = re_compile(r'[\/\\]python[0-9\.]+[\/\\]site-packages[\/\\]?$', RE_IGNORECASE)
ODBC_STRING_VARIABLE:REPattern = re_compile(r'\{([a-z_][a-z0-9_]*)\}|@([a-z0-9_]+)', RE_IGNORECASE) ODBC_STRING_VARIABLE:REPattern = re_compile(r'\{([a-z_][a-z0-9_]*)\}|@([a-z0-9_]+)', RE_IGNORECASE)
SQLITE_COMMENTS:REPattern = re_compile(r'--[^\r\n]*|\/\*(?:[^\*]|\*+[^\/\*]|[\r\n]+)*\*+\/')
# ORM # ORM
ORM_MAIN_BLOCKS:REPattern = re_compile(r'((?:(?!(?:(?:\r\n){2}|[\r\n]{2}))(?:.|[\r\n]))+)(?:(?:\r\n){2}|[\r\n]{2})((?:.+|[\r\n]+)+)') ORM_MAIN_BLOCKS:REPattern = re_compile(r'((?:(?!(?:(?:\r\n){2}|[\r\n]{2}))(?:.|[\r\n]))+)(?:(?:\r\n){2}|[\r\n]{2})((?:.+|[\r\n]+)+)')