#wip: Scrappers managers.
This commit is contained in:
parent
f1807f5f9d
commit
efa9b8b965
@ -622,9 +622,6 @@ export const ErrorsManager = (function(){
|
|||||||
"base_greater_than_alphabet"
|
"base_greater_than_alphabet"
|
||||||
];
|
];
|
||||||
|
|
||||||
/** @type {RegExp} */
|
|
||||||
ErrorsManager.RE_KEY = /^[a-z_][a-z0-9_]*$/i;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param {!(string|Array.<any|null>)} items
|
* @param {!(string|Array.<any|null>)} items
|
||||||
* @returns {string|Array.<any|null>}
|
* @returns {string|Array.<any|null>}
|
||||||
|
|||||||
@ -7,7 +7,7 @@ from time import time as timestamp, sleep
|
|||||||
from AnP.Interfaces.Application.AnPInterface import AnPInterface
|
from AnP.Interfaces.Application.AnPInterface import AnPInterface
|
||||||
from AnP.DSLs.ScrapDSL import (
|
from AnP.DSLs.ScrapDSL import (
|
||||||
Execute, Action, Open, Close, Switch, Wait, Do, Go, Get, ForEach, And, Or, Not,
|
Execute, Action, Open, Close, Switch, Wait, Do, Go, Get, ForEach, And, Or, Not,
|
||||||
If, While, DoWhile, Download, Selector, Condition
|
If, While, DoWhile, Download, Selector, Condition, Move, Click
|
||||||
)
|
)
|
||||||
from AnP.Utils.Common import Common
|
from AnP.Utils.Common import Common
|
||||||
from AnP.Utils.Checks import Check
|
from AnP.Utils.Checks import Check
|
||||||
@ -39,7 +39,9 @@ class BrowserAbstract(ModelAbstract):
|
|||||||
If : self._if,
|
If : self._if,
|
||||||
While : self._while,
|
While : self._while,
|
||||||
DoWhile : self.do_while,
|
DoWhile : self.do_while,
|
||||||
Download : self.download
|
Download : self.download,
|
||||||
|
Move : self.move,
|
||||||
|
Click : self.click
|
||||||
}
|
}
|
||||||
self.__print_actions:bool = Common.get_value("print_actions", inputs, True)
|
self.__print_actions:bool = Common.get_value("print_actions", inputs, True)
|
||||||
|
|
||||||
@ -393,4 +395,10 @@ class BrowserAbstract(ModelAbstract):
|
|||||||
break
|
break
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def download(self:Self, action:Download, shared:dict[str, Any|None], box:BrowserBoxModel) -> None:pass
|
def download(self:Self, action:Download, shared:dict[str, Any|None], box:BrowserBoxModel) -> None:pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def click(self:Self, action:Click, shared:dict[str, Any|None], box:BrowserBoxModel) -> None:pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def move(self:Self, action:Move, shared:dict[str, Any|None], box:BrowserBoxModel) -> None:pass
|
||||||
@ -204,7 +204,7 @@ class MouseAction(Action):
|
|||||||
def get_size(self:Self) -> Position:
|
def get_size(self:Self) -> Position:
|
||||||
return (
|
return (
|
||||||
self.size if self.size is not None else
|
self.size if self.size is not None else
|
||||||
Position(self.position.x, self.position.y) if isinstance(self.position, Selector) else
|
# Position(self.position.x, self.position.y) if isinstance(self.position, Selector) else
|
||||||
Position(10, 10))
|
Position(10, 10))
|
||||||
|
|
||||||
def get(self:Self) -> tuple[float, float]:
|
def get(self:Self) -> tuple[float, float]:
|
||||||
|
|||||||
@ -13,7 +13,7 @@ from seleniumwire.request import Request
|
|||||||
from selenium.webdriver.common.action_chains import ActionChains
|
from selenium.webdriver.common.action_chains import ActionChains
|
||||||
from AnP.DSLs.ScrapDSL import (
|
from AnP.DSLs.ScrapDSL import (
|
||||||
Open, Close, Switch, Go, Selector, XPath, CSS, Name, LinkText, Class, ID,
|
Open, Close, Switch, Go, Selector, XPath, CSS, Name, LinkText, Class, ID,
|
||||||
Get, Download, Action
|
Get, Download, Action, Click, Move, Position, MouseAction
|
||||||
)
|
)
|
||||||
from AnP.Interfaces.Application import AnPInterface
|
from AnP.Interfaces.Application import AnPInterface
|
||||||
from AnP.Abstracts.BrowserAbstract import BrowserAbstract
|
from AnP.Abstracts.BrowserAbstract import BrowserAbstract
|
||||||
@ -120,6 +120,15 @@ class SeleniumDriver(BrowserAbstract):
|
|||||||
def go(self:Self, action:Go, shared:dict[str, Any|None], box:BrowserBoxModel) -> None:
|
def go(self:Self, action:Go, shared:dict[str, Any|None], box:BrowserBoxModel) -> None:
|
||||||
self.__load(action, action.url, shared)
|
self.__load(action, action.url, shared)
|
||||||
|
|
||||||
|
def __move_to(self:Self, element:WebElement|Position|MouseAction) -> None:
|
||||||
|
if isinstance(element, WebElement):
|
||||||
|
self.browser.execute_script("arguments[0].scrollIntoView({block: 'center'});", element)
|
||||||
|
ActionChains(self.browser).move_to_element(element).perform()
|
||||||
|
elif isinstance(element, Position):
|
||||||
|
ActionChains(self.browser).move_by_offset(element.x, element.y).perform()
|
||||||
|
elif isinstance(element, MouseAction):
|
||||||
|
ActionChains(self.browser).move_by_offset(*element.get()).perform()
|
||||||
|
|
||||||
def get_selector(self:Self,
|
def get_selector(self:Self,
|
||||||
selector:Selector,
|
selector:Selector,
|
||||||
box:BrowserBoxModel,
|
box:BrowserBoxModel,
|
||||||
@ -156,13 +165,15 @@ class SeleniumDriver(BrowserAbstract):
|
|||||||
element:WebElement
|
element:WebElement
|
||||||
|
|
||||||
for element in item:
|
for element in item:
|
||||||
ActionChains(self.__browser).move_to_element(element).perform()
|
self.__move_to(element)
|
||||||
|
# ActionChains(self.__browser).move_to_element(element).perform()
|
||||||
shared[action.key].append(element.get_attribute(
|
shared[action.key].append(element.get_attribute(
|
||||||
"textContent" if action.attribute == "CDATA" else
|
"textContent" if action.attribute == "CDATA" else
|
||||||
action.attribute))
|
action.attribute))
|
||||||
|
|
||||||
else:
|
else:
|
||||||
ActionChains(self.__browser).move_to_element(item).perform()
|
self.__move_to(item)
|
||||||
|
# ActionChains(self.__browser).move_to_element(item).perform()
|
||||||
shared[action.key] = (item.get_attribute(
|
shared[action.key] = (item.get_attribute(
|
||||||
"textContent" if action.attribute == "CDATA" else
|
"textContent" if action.attribute == "CDATA" else
|
||||||
action.attribute))
|
action.attribute))
|
||||||
@ -246,4 +257,38 @@ class SeleniumDriver(BrowserAbstract):
|
|||||||
if action.path and (action.overwrite or not self.anp.files.exists(action.path)):
|
if action.path and (action.overwrite or not self.anp.files.exists(action.path)):
|
||||||
self.anp.files.save(action.path, data)
|
self.anp.files.save(action.path, data)
|
||||||
action.done = True
|
action.done = True
|
||||||
break
|
break
|
||||||
|
|
||||||
|
def move(self:Self, action:Move, shared:dict[str, Any|None], box:BrowserBoxModel) -> None:
|
||||||
|
|
||||||
|
element:WebElement|None = (
|
||||||
|
action.position if isinstance(action.position, WebElement) else
|
||||||
|
self.get_selector(action.position, box) if isinstance(action.position, Selector) else
|
||||||
|
None)
|
||||||
|
|
||||||
|
if element is None:
|
||||||
|
if isinstance(action.position, Sequence) and len(action.position) == 2 and all(isinstance(coord, (int, float)) for coord in action.position):
|
||||||
|
action.position = Position(action.position[0], action.position[1])
|
||||||
|
if isinstance(action.position, Position):
|
||||||
|
ActionChains(self.browser).move_by_offset(action.position.x, action.position.y).perform()
|
||||||
|
action.done = True
|
||||||
|
else:
|
||||||
|
action.done = False
|
||||||
|
else:
|
||||||
|
self.__move_to(MouseAction(
|
||||||
|
Position(element.location["x"], element.location["y"]),
|
||||||
|
action.margin,
|
||||||
|
Position(element.size["width"], element.size["height"])
|
||||||
|
))
|
||||||
|
action.done = True
|
||||||
|
|
||||||
|
def click(self:Self, action:Click, shared:dict[str, Any|None], box:BrowserBoxModel) -> None:
|
||||||
|
|
||||||
|
move_action:Move = Move(action.position, action.margin, action.size)
|
||||||
|
|
||||||
|
self.move(move_action, shared, box)
|
||||||
|
if move_action.done:
|
||||||
|
ActionChains(self.browser).click().perform()
|
||||||
|
action.done = True
|
||||||
|
else:
|
||||||
|
action.done = False
|
||||||
@ -6,7 +6,7 @@ from requests import get, Response
|
|||||||
from parsel import Selector as ParseSelector, SelectorList as ParseSelectorList
|
from parsel import Selector as ParseSelector, SelectorList as ParseSelectorList
|
||||||
from AnP.DSLs.ScrapDSL import (
|
from AnP.DSLs.ScrapDSL import (
|
||||||
Selector, CSS, XPath, Name, LinkText, Class, ID, Open,
|
Selector, CSS, XPath, Name, LinkText, Class, ID, Open,
|
||||||
Go, Get, Download, Action
|
Go, Get, Download, Action, Click, Move
|
||||||
)
|
)
|
||||||
from AnP.Abstracts.BrowserAbstract import BrowserAbstract
|
from AnP.Abstracts.BrowserAbstract import BrowserAbstract
|
||||||
from AnP.Models.BrowserTabsModel import BrowserBoxModel
|
from AnP.Models.BrowserTabsModel import BrowserBoxModel
|
||||||
@ -113,4 +113,10 @@ class XMLScrapDriver(BrowserAbstract):
|
|||||||
|
|
||||||
except Exception as exception:
|
except Exception as exception:
|
||||||
print(exception)
|
print(exception)
|
||||||
action.exception = exception
|
action.exception = exception
|
||||||
|
|
||||||
|
def click(self:Self, action:Click, shared:dict[str, Any|None], box:BrowserBoxModel) -> None:
|
||||||
|
action.done = True
|
||||||
|
|
||||||
|
def move(self:Self, action:Move, shared:dict[str, Any|None], box:BrowserBoxModel) -> None:
|
||||||
|
action.done = True
|
||||||
@ -62,9 +62,9 @@ class ControllersManager:
|
|||||||
controller if Check.is_dictionary(controller) else
|
controller if Check.is_dictionary(controller) else
|
||||||
None))
|
None))
|
||||||
|
|
||||||
def execute(self:Self, key:str, method:str, request:RequestModel) -> Any|None:
|
def execute(self:Self, key:str, method:str, *arguments:Any) -> Any|None:
|
||||||
if key in self.__controllers and hasattr(self.__controllers[key], method):
|
if key in self.__controllers and hasattr(self.__controllers[key], method):
|
||||||
return getattr(self.__controllers[key], method)(request)
|
return getattr(self.__controllers[key], method)(*arguments)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def get(self:Self, key:str, Type:type[ControllerType]) -> ControllerType|None:
|
def get(self:Self, key:str, Type:type[ControllerType]) -> ControllerType|None:
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user