From bc0a86900a49a3d8a54b888e3764e7f5a1ad8ef8 Mon Sep 17 00:00:00 2001 From: KyMAN <0kyman0@gmail.com> Date: Wed, 1 Jul 2026 07:46:29 +0200 Subject: [PATCH] #wip: ScrapDSL and Drivers. --- JSON/AnP.settings.json | 43 --- JSON/I18N/AnP.i18n.espanol.json | 10 +- .../ecma/Components/AIChatComponent.ecma.js | 165 ++++++---- Public/ecma/Managers/SessionsManager.ecma.js | 79 ++++- Public/ecma/Models/AIChatModel.ecma.js | 196 +++++++++++ Public/ecma/Models/SessionModel.ecma.js | 94 +++++- Public/json/i18n/AnP.i18n.espanol.json | 3 +- Python/AnP/Abstracts/BrowserAbstract.py | 181 ++++++++++ Python/AnP/Application/AnP.py | 5 + Python/AnP/DSLs/HTMLDLS.py | 2 + Python/AnP/DSLs/ScrapDSL.py | 166 ++++++++++ Python/AnP/Drivers/MySQLDriver.py | 2 + Python/AnP/Drivers/SQLServerDriver.py | 2 + Python/AnP/Drivers/SQLiteDriver.py | 2 + Python/AnP/Drivers/SeleniumDriver.py | 12 + Python/AnP/Drivers/XMLScrapDriver.py | 309 ++++++++++++++++++ .../Abstracts/BrowserAbstractInterface.py | 72 ++++ .../Abstracts/FilesAbstractInterface.py | 13 +- .../Abstracts/HTTPServersAbstractInterface.py | 3 +- .../WebSocketServersAbstractInterface.py | 3 +- .../Interfaces/Application/AnPInterface.py | 2 + .../Managers/BrowsersManagerInterface.py | 27 ++ Python/AnP/Managers/BrowsersManager.py | 69 ++++ Python/AnP/Managers/ControllersManager.py | 4 + Python/AnP/Managers/DispatchesManager.py | 4 + Python/AnP/Models/CoordenateModel.py | 13 + Python/AnP/Models/PositionModel.py | 30 ++ Python/AnP/Models/XMLScrapTabModel.py | 17 + Python/AnP/Utils/Common.py | 12 +- Python/base.py | 4 +- 30 files changed, 1422 insertions(+), 122 deletions(-) create mode 100644 Public/ecma/Models/AIChatModel.ecma.js create mode 100644 Python/AnP/Abstracts/BrowserAbstract.py create mode 100644 Python/AnP/DSLs/HTMLDLS.py create mode 100644 Python/AnP/DSLs/ScrapDSL.py create mode 100644 Python/AnP/Drivers/MySQLDriver.py create mode 100644 Python/AnP/Drivers/SQLServerDriver.py create mode 100644 Python/AnP/Drivers/SQLiteDriver.py create mode 100644 Python/AnP/Drivers/SeleniumDriver.py create mode 100644 Python/AnP/Drivers/XMLScrapDriver.py create mode 100644 Python/AnP/Interfaces/Abstracts/BrowserAbstractInterface.py create mode 100644 Python/AnP/Interfaces/Managers/BrowsersManagerInterface.py create mode 100644 Python/AnP/Managers/BrowsersManager.py create mode 100644 Python/AnP/Models/CoordenateModel.py create mode 100644 Python/AnP/Models/PositionModel.py create mode 100644 Python/AnP/Models/XMLScrapTabModel.py diff --git a/JSON/AnP.settings.json b/JSON/AnP.settings.json index fab2870..393a3ca 100644 --- a/JSON/AnP.settings.json +++ b/JSON/AnP.settings.json @@ -200,24 +200,9 @@ "AnP_HTTPAbstracts_end" : null, "AnP_WebSocketsServersManager_start" : null, - "default_web_sockets_servers" : { - "anp" : { - "type" : "WebSocketServerDriver", - "host" : "", - "port" : 18765 - } - }, "AnP_WebSocketsServersManager_end" : null, "AnP_HTTPServersManager_start" : null, - "default_http_servers" : { - "anp" : { - "type" : "HTTPServerDriver", - "type2" : "HTTPSocketServerDriver", - "host" : "", - "port" : 18000 - } - }, "AnP_HTTPServersManager_end" : null, "AnP_PseudoLoRAsManager_start" : null, @@ -225,34 +210,6 @@ "AnP_PseudoLoRAsManager_end" : null, "AnP_AIInterpretersManager_start" : null, - "default_ai_interpreters" : { - "anp_titles" : { - "type" : "OllamaDriver", - "url" : "http://localhost:11434/api/generate/", - "model" : "gemma3:1b", - "pool" : "anp", - "format" : { - "type": "array", - "items": { - "type": "boolean" - } - }, - "stream" : false, - "think" : false, - "temperature" : 0.0, - "allow_contexts" : true - }, - "anp_responses" : { - "type" : "OllamaDriver", - "url" : "http://localhost:11434/api/generate/", - "model" : "gemma4:e4b", - "stream" : true, - "pool" : "anp", - "think" : false, - "temperature" : 0.7, - "allow_contexts" : true - } - }, "AnP_AIInterpretersManager_end" : null, "AnP_TitlesManager_start" : null, diff --git a/JSON/I18N/AnP.i18n.espanol.json b/JSON/I18N/AnP.i18n.espanol.json index 83f5bf7..9e81a69 100644 --- a/JSON/I18N/AnP.i18n.espanol.json +++ b/JSON/I18N/AnP.i18n.espanol.json @@ -61,7 +61,15 @@ "AnP_CloneController_start" : null, "anp_clone_controller_file_cloned" : "El archivo '{file}' ha sido clonado desde '{from_path}' a '{to_path}'.", "anp_clone_controller_file_deleted" : "El archivo '{file}' ha sido eliminado de '{to_path}'.", - "AnP_CloneController_end" : null + "AnP_CloneController_end" : null, + + "AnP_BrowserAbstract_start" : null, + "anp_browser_action_exception" : "Hubo una excepción al intentar ejecutar la acción '{i}' de tipo '{action}' en el navegador '{key}'.", + "anp_browser_action_done" : "La acción '{i}' de tipo '{action}' ha sido ejecutada correctamente en el navegador '{key}'.", + "anp_browser_action_not_done" : "La acción '{i}' de tipo '{action}' no ha sido ejecutada correctamente en el navegador '{key}'.", + "anp_browser_action_not_supported" : "La acción '{i}' de tipo '{action}' no es compatible con el navegador '{key}'.", + "anp_browser_action_not_action" : "El elemento '{i}' no es una acción válida en el navegador '{key}'.", + "AnP_BrowserAbstract_end" : null } } \ No newline at end of file diff --git a/Public/ecma/Components/AIChatComponent.ecma.js b/Public/ecma/Components/AIChatComponent.ecma.js index 88762d0..4adc788 100644 --- a/Public/ecma/Components/AIChatComponent.ecma.js +++ b/Public/ecma/Components/AIChatComponent.ecma.js @@ -56,6 +56,76 @@ export const AIChatComponent = (function(){ return item; }; + /** + * @param {!string} id + * @param {!string} type + * @param {?string} [content = ""] + * @param {?string} [mode = "waiting"] + * @param {?(Object.|Array.)} [inputs = null] + * @returns {Array.} + * @access private + */ + const build_message = (id, type, content = "", mode = "waiting", inputs = null) => { + + /** @type {[string, string, string]} */ + const [html, raw_html, md] = format(content), + /** @type {number} */ + date_from = Common.get_value("date_from", inputs) || Date.now(); + + return Fieldset({ + class : "message", + data_id : id, + data_type : type, + data_ok : true, + data_status : mode, + data_done : mode == "done", + data_mode : "html" + }, [ + anp.components.i18n(type, "legend"), + Div({class : "message-box html-content"}, html), + Pre({class : "message-box raw-html-content"}, raw_html), + Pre({class : "message-box md-content"}, md), + Nav({class : "view buttons"}, [ + anp.components.button("html", set_view), + anp.components.button("raw_html", set_view), + anp.components.button("md", set_view), + ]), + UL({class : "data"}, [ + build_data_icon("status", mode), + build_data_item("ok"), + build_data_item("done"), + build_data_item("date_from", date_from), + build_data_item("date_to", Common.get_value("date_to", inputs) || date_from), + build_data_item("time", 0), + build_data_item("length", content.length), + build_data_item("response_tokens", content.replace(/(?:[^a-z0-9]+|[\r\n]+)+/gi, " ").trim().split(" ").length) + ]) + ]); + }; + + /** + * @param {?string} session + * @param {!string} name + * @returns {Array.} + * @access private + */ + const build_new_conversation = (session, name) => { + + const session_key = session + "-" + name; + /** @type {string} */ + conversation = anp.unique_keys.get(); + + cache[cache_key].conversations[conversation] = []; + + document.querySelectorAll(".aichat[") + + return Article({ + class : "messages", + data_conversation : conversation, + data_selected : true + }); + }; + /** * @param {?(Object.|Array.)} [inputs = null] * @returns {Array.} @@ -64,20 +134,49 @@ export const AIChatComponent = (function(){ this.build = (inputs = null) => { /** @type {string} */ - const name = Common.get_value("name", inputs, "aichat"); + const name = Common.get_value("name", inputs, "aichat"), + /** @type {string|null} */ + session = Common.get_value("session", inputs, null), + /** @type {string} */ + cache_key = session + "-" + name; + + if(session){ + cache[cache_key] = { + selected : conversations[conversations.length - 1], + conversations : conversations.reduce((dictionary, conversation) => { + + dictionary[conversation] = []; + + return dictionary; + }, {}) + }; + Common.preload("[data-name=" + name + "]", (chats, asynchronous, ok) => { + if(ok){ + + const data = Common.data_decode(anp.sessions.get_value(session, name)); + + if(data){ + cache[cache_key] = data; + return; + }; + + Common.HTML(chats.querySelector(".conversations"), build_new_conversation(session, name)); + + }; + }); + }; return Fieldset({ class : "aichat", - data_name : Common.get_value("name", inputs, "aichat") + data_name : name, + data_session : session }, [ anp.components.i18n(name, "legend"), - Section({class : "conversations"}, [ - Article({ - class : "messages", - data_conversation : anp.unique_keys.get(), - data_selected : true - }) - ]), + Section({class : "conversations"}, conversations.map(conversation => Article({ + class : "messages", + data_conversation : conversation, + data_selected : true + }))), Form({ method : "post", action : "#", @@ -172,52 +271,6 @@ export const AIChatComponent = (function(){ return [html, html.replace(//g, ">"), content]; }; - /** - * @param {!string} id - * @param {!string} type - * @param {?string} [content = ""] - * @param {?string} [mode = "waiting"] - * @returns {HTMLFielsetElement} - * @access private - */ - const build_message = (id, type, content = "", mode = "waiting") => { - - /** @type {[string, string, string]} */ - const [html, raw_html, md] = format(content), - /** @type {number} */ - date = Date.now(); - - return Fieldset({ - class : "message", - data_id : id, - data_type : type, - data_ok : true, - data_status : mode, - data_done : mode == "done", - data_mode : "html" - }, [ - anp.components.i18n(type, "legend"), - Div({class : "message-box html-content"}, html), - Pre({class : "message-box raw-html-content"}, raw_html), - Pre({class : "message-box md-content"}, md), - Nav({class : "view buttons"}, [ - anp.components.button("html", set_view), - anp.components.button("raw_html", set_view), - anp.components.button("md", set_view), - ]), - UL({class : "data"}, [ - build_data_icon("status", mode), - build_data_item("ok"), - build_data_item("done"), - build_data_item("date_from", date), - build_data_item("date_to", date), - build_data_item("time", 0), - build_data_item("length", content.length), - build_data_item("response_tokens", content.replace(/(?:[^a-z0-9]+|[\r\n]+)+/gi, " ").trim().split(" ").length) - ]) - ]); - }; - /** * * @param {!HTMLElement} item @@ -238,8 +291,6 @@ export const AIChatComponent = (function(){ if(text){ - console.log([item, item.querySelector("[name=message]"), item.querySelector("[name=message]").parentNode.parentNode.getAttribute("data-connection")]); - /** @type {string} */ const data_id = anp.unique_keys.get(), /** @type {[string, string, string]} */ diff --git a/Public/ecma/Managers/SessionsManager.ecma.js b/Public/ecma/Managers/SessionsManager.ecma.js index 39c5e71..df1aec2 100644 --- a/Public/ecma/Managers/SessionsManager.ecma.js +++ b/Public/ecma/Managers/SessionsManager.ecma.js @@ -28,7 +28,9 @@ export const SessionsManager = (function(){ /** @type {SessionsManager} */ const self = this, /** @type {Object.} */ - sessions = {}; + sessions = {}, + /** @type {Object.} */ + keys = {}; /** * @returns {void} @@ -36,17 +38,31 @@ export const SessionsManager = (function(){ */ const constructor = () => {}; + /** + * @param {!(string|number)} id + * @returns {SessionModel|null} + * @access public + */ + const get = id => sessions[id] || sessions[keys[id]] || null; + /** * @param {!(Object.|Array.)} inputs - * @return {number} + * @return {number|null} * @access public */ this.set = inputs => { + const key = Common.get_value("key", inputs, null); + + if(keys[key]) + return null; + /** @type {number} */ let i; while(sessions[i = Math.random() * (1 << 28) | 0]); + if(key) + keys[key] = i; return (sessions[i] = new SessionModel(i, inputs)).i; }; @@ -57,16 +73,24 @@ export const SessionsManager = (function(){ * @returns {boolean} * @access public */ - this.check_permissions = (ids, permissions) => Common.get_array(ids).some(id => ( - sessions[id] && sessions[id].check_permissions(permissions) - )); + this.check_permissions = (ids, permissions) => Common.get_array(ids).some(id => { + + /** @type {SessionModel|null} */ + const session = get(id); + + return session && session.check_permissions(permissions) + }); /** - * @param {!number} id + * @param {!(string|number)} id * @returns {boolean} * @access public */ this.close = id => { + if(keys[id]){ + id = keys[id]; + delete keys[id]; + }; if(sessions[id]){ delete sessions[id]; return true; @@ -74,6 +98,49 @@ export const SessionsManager = (function(){ return false; }; + /** + * @param {!(string|number)} id + * @param {!string} key + * @returns {boolean} + * @access public + */ + this.has_value = (id, key) => { + + /** @type {SessionModel|null} */ + const session = get(id); + + return session ? session.has(key) : false; + }; + + /** + * @param {!(string|number)} id + * @param {!string} key + * @returns {any|null} + * @access public + */ + this.get_value = (id, key) => { + + /** @type {SessionModel|null} */ + const session = get(id); + + return session ? session.get(key) : null; + }; + + /** + * @param {!(string|number)} id + * @param {!string} key + * @param {?any} [value = null] + * @returns {boolean} + * @access public + */ + this.set_value = (id, key, value) => { + + /** @type {SessionModel|null} */ + const session = get(id); + + return session ? session.set(key, value) : null; + }; + constructor(); }; diff --git a/Public/ecma/Models/AIChatModel.ecma.js b/Public/ecma/Models/AIChatModel.ecma.js new file mode 100644 index 0000000..7693223 --- /dev/null +++ b/Public/ecma/Models/AIChatModel.ecma.js @@ -0,0 +1,196 @@ +"use strict"; + +import {Event} from "../Application/Event.ecma.js"; +import {Common} from "../Utils/Common.ecma.js"; + +/** + * @class AIChatMessageModel + * @constructor + * @param {!AIChatConversationModel} conversation + * @param {!(Object.|Array.)} inputs + * @return {void} + * @access public + * @static + */ +export const AIChatMessageModel = (function(){ + + /** + * @constructs AIChatMessageModel + * @param {!AIChatConversationModel} conversation + * @param {!(Object.|Array.)} inputs + * @return {void} + * @access private + * @static + */ + const AIChatMessageModel = function(conversation, inputs){ + /** @type {AIChatConversationModel} */ + this.conversation = conversation; + /** @type {string} */ + this.status = Common.get_value("status", inputs, "unloaded"); + /** @type {string|null} */ + this.id = Common.get_value("id", inputs); + /** @type {string} */ + this.role = Common.get_value("role", inputs, "unknown"); + /** @type {boolean} */ + this.ok = Common.get_value("ok", inputs, false); + /** @type {boolean} */ + this.done = Common.get_value("done", inputs, false); + /** @type {string} */ + this.message = (Common.get_value("message", inputs, "")); + /** @type {number} */ + this.tokens = Common.get_value("tokens", inputs, 0); + /** @type {number} */ + this.date_from = Common.get_value("date_from", inputs, Date.now()); + /** @type {number} */ + this.date_to = Common.get_value("date_to", inputs, Date.now()); + }; + + return AIChatMessageModel; +})(); + +/** + * @class AIChatConversationModel + * @constructor + * @param {!AIChatModel} chat + * @param {!(Object.|Array.)} inputs + * @return {void} + * @access public + * @static + */ +export const AIChatConversationModel = (function(){ + + /** + * @constructs AIChatConversationModel + * @param {!AIChatModel} chat + * @param {!(Object.|Array.)} inputs + * @return {void} + * @access private + * @static + */ + const AIChatConversationModel = function(chat, inputs){ + /** @type {AIChatModel} */ + this.chat = chat; + /** @type {string} */ + this.id = Common.get_value("id", inputs); + /** @type {Array.} */ + this.messages = Common.get_value("messages", inputs, []).map(message => new AIChatMessageModel(this, message)); + /** @type {boolean} */ + this.selected = Common.get_value("selected", inputs, true); + }; + + return AIChatConversationModel; +})(); + +/** + * @class AIChatModel + * @constructor + * @return {void} + * @access private + * @static + */ +export const AIChatModel = (function(){ + + /** + * @constructs AIChatModel + * @return {void} + * @access private + * @static + */ + const AIChatModel = function(){ + + /** @type {AIChatModel} */ + const self = this; + + /** @type {Array.} */ + this.conversations = []; + /** @type {AIChatConversationModel|null} */ + this.selected_conversation = null; + + /** @type {Event} */ + this.on_change = new Event(); + + /** + * @returns {void} + * @access private + */ + const constructor = () => { + + self.on_change.add(update_changes); + + }; + + /** + * @param {!AIChatMessageModel} model + * @returns {void} + * @access private + */ + const update_changes = model => { + + /** @type {HTMLElement|null} */ + const message = document.querySelector(".aichat [data-conversation='" + model.conversation.id + "'] .aichat-message[data-id='" + model.id + "']"); + + }; + + /** + * @param {!string} conversation + * @param {!string} id + * @returns {AIChatMessageModel|null} + * @access private + */ + const get_message = (conversation, id) => { + + /** @type {AIChatConversationModel|null} */ + const conversation_model = self.conversations.find(model => model.id === conversation); + + if(conversation_model) + return conversation_model.messages.find(model => model.id === id) || null; + return null; + }; + + /** + * @param {!string} message + * @param {!boolean} [done = false] + * @returns {boolean} + * @access public + */ + this.update = (conversation, id, message, done = false) => { + + /** @type {AIChatMessageModel|null} */ + let model = get_message(conversation, id); + + if(!model) + return false; + + model.message += message; + model.tokens ++; + model.done = done; + conversation_model.chat.on_change.execute(model); + + return true; + }; + + /** + * @param {!string} status + * @returns {boolean} + * @access public + */ + this.set_status = status => { + + /** @type {AIChatMessageModel|null} */ + let model = get_message(conversation, id); + + if(!model) + return false; + + model.status = status; + conversation_model.chat.on_change.execute(model); + + return true; + }; + + constructor(); + + }; + + return AIChatModel; +})(); \ No newline at end of file diff --git a/Public/ecma/Models/SessionModel.ecma.js b/Public/ecma/Models/SessionModel.ecma.js index dca5e25..8166af8 100644 --- a/Public/ecma/Models/SessionModel.ecma.js +++ b/Public/ecma/Models/SessionModel.ecma.js @@ -23,13 +23,19 @@ export const SessionModel = (function(){ const SessionModel = function(i, inputs){ /** @type {SessionModel} */ - const self = this; + const self = this, + /** @type {Object.} */ + data = {}; /** @type {number|null} */ let id = null, /** @type {Array.} */ permissions = [], /** @type {number} */ - last_check = 0; + last_check = 0, + /** @type {string|null} */ + cookie_key = null, + /** @type {integer|null} */ + timeout = null; /** @type {string|null} */ this.nick = null; @@ -43,11 +49,36 @@ export const SessionModel = (function(){ * @access private */ const constructor = () => { - id = Common.get_value("id", inputs, null); - permissions = Common.get_value("permissions", inputs, []); - self.nick = Common.get_value("nick", inputs, null); - self.type = Common.get_value("type", inputs, self.type); + + /** @type {string|null} */ + const cookie = anp.cookies.get(cookie_key = Common.get_value("cookie", inputs, null)); + + if(cookie){ + + /** @type {string} */ + let raw_permissions, + /** @type {string} */ + raw_data, + /** @type {string} */ + raw_type, + /** @type {string} */ + raw_id; + + [raw_id, self.nick, raw_type, raw_permissions, raw_data] = cookie.split("|"); + permissions = raw_permissions.split(","); + Object.assign(Common.data_encode(raw_data)).forEach(([key, value]) => self.set(key, value, false)); + self.type = Number(raw_type) || SessionModel.LOCAL; + id = Number(raw_id) || null; + + }else{ + id = Common.get_value("id", inputs, null); + permissions = Common.get_value("permissions", inputs, []); + self.nick = Common.get_value("nick", inputs, null); + self.type = Number(Common.get_value("type", inputs, self.type)) || SessionModel.LOCAL; + }; last_check = Date.now(); + timeout = Common.get_value(["session_timeout", "timeout"], inputs, 300000); + }; /** @@ -57,6 +88,57 @@ export const SessionModel = (function(){ */ this.check_permissions = permissions => permissions.some(self.permissions.includes); + /** + * @param {!string} key + * @returns {boolean} + * @access public + */ + this.has = key => data[key] !== undefined; + + /** + * @param {!string} key + * @param {?any} [value = null] + * @param {!boolean} [overwrite = true] + * @returns {boolean} + * @access public + */ + this.set = (key, value = null, overwrite = true) => { + if(Check.is_key(key) && (overwrite || !self.has(key))){ + data[key] = value; + return true; + }; + return false; + }; + + /** + * @param {!string} key + * @returns {any|null} + * @access public + */ + this.get = key => self.has(key) ? data[key] : null; + + /** + * @returns {boolean} + * @access public + */ + this.save = () => { + if(cookie_key){ + anp.cookies.set(cookie_key, Common.data_encode([ + id, + self.nick, + self.type, + permissions.join(","), + Common.data_encode(data) + ]).join("|"), { + overwrite : true, + expires : new Date(Date.now() + timeout), + path : "/" + }); + return true; + }; + return false; + }; + constructor(); }; diff --git a/Public/json/i18n/AnP.i18n.espanol.json b/Public/json/i18n/AnP.i18n.espanol.json index 029e467..3ca8db3 100644 --- a/Public/json/i18n/AnP.i18n.espanol.json +++ b/Public/json/i18n/AnP.i18n.espanol.json @@ -32,6 +32,7 @@ "raw_html" : "Código HTML", "minimum" : "Mínimo", "maximum" : "Máximo", - "ai" : "IA" + "ai" : "IA", + "anpbot" : "AnPBot" } } \ No newline at end of file diff --git a/Python/AnP/Abstracts/BrowserAbstract.py b/Python/AnP/Abstracts/BrowserAbstract.py new file mode 100644 index 0000000..53b1f9c --- /dev/null +++ b/Python/AnP/Abstracts/BrowserAbstract.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from typing import Optional, Self, Any, Callable, Sequence, TypeVar +from abc import abstractmethod +from AnP.Interfaces.Application.AnPInterface import AnPInterface +from AnP.DSLs.ScrapDSL import ( + Execute, Action, Open, Close, Switch, Wait, Do, Go, Get, ForEach, And, Or, Not, + If, While, DoWhile, Download +) +from AnP.Utils.Common import Common +from AnP.Abstracts.ModelAbstract import ModelAbstract + +T = TypeVar("T") + +class BrowserAbstract(ModelAbstract): + + def __init__(self:Self, anp:AnPInterface, key:str, TabType:T, inputs:Optional[dict[str, Any|None]|Sequence[Any|None]] = None) -> None: + self.anp:AnPInterface = anp + self.key:str = key + self.working:bool = True + self.tabs:dict[str, T] = {} + self.tab_selected:str = "" + self.human_time:float|int|Sequence[float|int] = Common.get_value("human_time", inputs, 0.0) + self.actions:dict[Action, Callable[[Action, dict[str, Any|None], Optional[Any]], None]] = { + Execute : self.execute, + Open : self.open, + Close : self.close, + Switch : self.switch, + Wait : self.wait, + Do : self.do, + Go : self.go, + Get : self.get, + ForEach : self.foreach, + And : self._and, + Or : self._or, + Not : self._not + } + + def start(self:Self) -> None: + self.working = True + + def close(self:Self) -> None: + self.working = False + self.tabs.clear() + self.tab_selected = "" + + def is_working(self:Self) -> bool: + return self.working and self.anp.working() + + @abstractmethod + def run(self:Self, actions:Sequence[Action], shared:dict[str, Any|None] = {}) -> dict[str, Any|None]:pass + + @abstractmethod + def execute(self:Self, action:Execute, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def open(self:Self, action:Open, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def close(self:Self, action:Close, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def switch(self:Self, action:Switch, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + def wait(self:Self, action:Wait, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None: + if action.is_fixed: + self.anp.wait(action.time) + else: + if action.is_range: + self.anp.wait(Common.random_number(*action.time[0])) + else: + self.anp.wait(Common.random(*action.time)) + action.done = True + + def do(self:Self, action:Do, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None: + + subaction:Action + i:int + + action.done = True + + for i, subaction in enumerate(action.actions): + if not self.anp.working(): + action.done = False + return + + if isinstance(subaction, Action): + subaction.done = False + subaction.exception = None + if subaction.__class__ in self.actions: + try: + + ok:bool + + self.actions[subaction.__class__](subaction, shared, last_item) + ok = subaction.done and subaction.exception is None + + if not ok and not subaction.ignore_fail: + if subaction.exception is None: + subaction.print_errors and self.anp.print("warning", "anp_browser_action_not_done", { + "key" : self.key, + "action": subaction.__class__.__name__, + "i" : i + }) + else: + self.anp.exception(subaction.exception, "anp_browser_action_exception", { + "key" : self.key, + "action": subaction.__class__.__name__, + "i" : i + }) + action.done = False + break + + if subaction.wait: + self.wait(Wait(self.human_time, True)) + + ok and subaction.print_ok and self.anp.print("info", "anp_browser_action_done", { + "key" : self.key, + "action": subaction.__class__.__name__, + "i" : i + }) + + except Exception as exception: + subaction.exception = exception + subaction.print_errors and self.anp.exception(exception, "anp_browser_action_exception", { + "key" : self.key, + "action": subaction.__class__.__name__, + "i" : i + }) + if not subaction.ignore_fail: + action.done = False + break + + else: + subaction.print_errors and self.anp.print("warning", "anp_browser_action_not_supported", { + "key" : self.key, + "action": subaction.__class__.__name__, + "i" : i + }) + if not subaction.ignore_fail: + action.done = False + break + else: + self.anp.print("warning", "anp_browser_action_not_action", { + "key" : self.key, + "i" : i + }) + action.done = False + break + + + @abstractmethod + def go(self:Self, action:Go, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def get(self:Self, action:Get, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def foreach(self:Self, action:ForEach, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def _and(self:Self, action:And, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> bool:pass + + @abstractmethod + def _or(self:Self, action:Or, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> bool:pass + + @abstractmethod + def _not(self:Self, action:Not, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> bool:pass + + @abstractmethod + def _if(self:Self, action:If, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def _while(self:Self, action:While, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def do_while(self:Self, action:DoWhile, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def download(self:Self, action:Download, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass \ No newline at end of file diff --git a/Python/AnP/Application/AnP.py b/Python/AnP/Application/AnP.py index 7d95c22..a66e993 100644 --- a/Python/AnP/Application/AnP.py +++ b/Python/AnP/Application/AnP.py @@ -24,6 +24,7 @@ from AnP.Managers.WebSocketServersManager import WebSocketServersManager from AnP.Managers.HTTPServersManager import HTTPServersManager from AnP.Managers.PseudoLoRAsManager import PseudoLoRAsManager from AnP.Managers.AIInterpretersManager import AIInterpretersManager +from AnP.Managers.BrowsersManager import BrowsersManager from AnP.Utils.Common import Common from AnP.Utils.Patterns import RE @@ -62,6 +63,7 @@ class AnP: self.http_servers:HTTPServersManager = HTTPServersManager(self) self.pseudoloras:PseudoLoRAsManager = PseudoLoRAsManager(self) self.ai_interpreters:AIInterpretersManager = AIInterpretersManager(self) + self.browsers:BrowsersManager = BrowsersManager(self) def update(self:Self) -> None: self.settings.update() @@ -81,6 +83,7 @@ class AnP: self.http_servers.update() self.pseudoloras.update() self.ai_interpreters.update() + self.browsers.update() def reset(self:Self) -> None: self.settings.reset() @@ -98,6 +101,7 @@ class AnP: self.web_socket_servers.reset() self.http_servers.reset() self.ai_interpreters.reset() + self.browsers.reset() def close(self:Self) -> None: self.__working = False @@ -105,6 +109,7 @@ class AnP: self.ai_interpreters.close() self.web_socket_servers.close() self.http_servers.close() + self.browsers.close() def working(self:Self) -> bool: return self.__working diff --git a/Python/AnP/DSLs/HTMLDLS.py b/Python/AnP/DSLs/HTMLDLS.py new file mode 100644 index 0000000..1f776fc --- /dev/null +++ b/Python/AnP/DSLs/HTMLDLS.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- \ No newline at end of file diff --git a/Python/AnP/DSLs/ScrapDSL.py b/Python/AnP/DSLs/ScrapDSL.py new file mode 100644 index 0000000..a3970d7 --- /dev/null +++ b/Python/AnP/DSLs/ScrapDSL.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from typing import Self, Sequence, Any, Optional, Callable +from re import Pattern as REPattern, Match as REMatch + +class Action: + def __init__(self:Self) -> None: + self.exception:Exception|None = None + self.print_errors:bool = True + self.print_ok:bool = False + self.done:bool = False + self.wait:bool = True + +class Execute(Action): + def __init__(self:Self, callback:Callable[[dict[str, Any|None]], None]) -> None: + super().__init__() + self.callback:Callable[[dict[str, Any|None]], None] = callback + +class Open(Action): + def __init__(self:Self, key:str, url:Optional[str] = None) -> None: + super().__init__() + self.key:str = key + self.url:str|None = url + +class Close(Action): + def __init__(self:Self, key:Optional[str] = None) -> None: + super().__init__() + self.key:str|None = key + +class Switch(Action): + def __init__(self:Self, key:str) -> None: + super().__init__() + self.key:str = key + +class Wait(Action): + def __init__(self:Self, time:float|int|Sequence[float|int], is_range:bool = True) -> None: + super().__init__() + self.time:float|int|Sequence[float|int] = time + self.is_fixed:bool = isinstance(time, (float, int)) + self.is_range:bool = is_range + +class Do(Action): + def __init__(self:Self, *actions:Action) -> None: + super().__init__() + self.actions:tuple[Action, ...] = actions + +class Go(Action): + def __init__(self:Self, url:str, tab:Optional[str] = None) -> None: + super().__init__() + self.url:str = url + self.tab:str|None = tab + +class Selector(Action): + def __init__(self:Self, + selector:str, + multiple:bool = False + ) -> None: + super().__init__() + self.selector:str = selector + self.multiple:bool = multiple + +class XPath(Selector):pass + +class CSS(Selector):pass + +class ID(Selector):pass + +class Class(Selector):pass + +class LinkText(Selector):pass + +class Name(Selector):pass + +class Get(Action): + def __init__(self:Self, + key:str, + attribute:str = "CDATA", + selector:Optional[Selector] = None, + reset:bool = False, + pattern:tuple[Optional[REPattern], str|Callable[[REMatch], str]] = None + ) -> None: + super().__init__() + + self.key:str = key + self.attribute:str = attribute + self.selector:Selector|None = selector + self.reset:bool = reset + self.pattern:REPattern|None = None + self.matches:str|Callable[[REMatch], str]|None = None + + if pattern is not None: + self.pattern, self.matches = pattern + +class ForEach(Action): + def __init__(self:Self, + items:Selector|Sequence[Any|None], + actions:Sequence[Action], + callback:Optional[Callable[[Any|None], None]] = None + ) -> None: + super().__init__() + self.items:Selector|Sequence[Any|None] = items + self.actions:Sequence[Action] = actions + self.callback:Callable[[Any|None], None]|None = callback + +class Condition(Action): + def __init__(self:Self, conditions:Sequence[Callable[[dict[str, Any|None]], bool]]|Selector|Self) -> None: + super().__init__() + self.conditions:Sequence[Callable[[dict[str, Any|None]], bool]|Selector|Condition|bool] = conditions + self.ok:bool = False + +class And(Condition):pass + +class Or(Condition):pass + +class Not(Action): + def __init__(self:Self, condition:Callable[[dict[str, Any|None]], bool]|Selector|Condition|bool) -> None: + super().__init__([condition]) + self.condition:Callable[[dict[str, Any|None]], bool]|Selector|Condition|bool = condition + self.ok:bool = False + +class If(Action): + def __init__(self:Self, + condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool, + if_actions:Optional[Sequence[Action]] = None, + else_actions:Optional[Sequence[Action]] = None + ) -> None: + super().__init__() + self.condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool = condition + self.if_actions:Sequence[Action]|None = if_actions + self.else_actions:Sequence[Action]|None = else_actions + +class While(Action): + def __init__(self:Self, + condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool, + actions:Sequence[Action] + ) -> None: + super().__init__() + self.condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool = condition + self.actions:Sequence[Action] = actions + +class DoWhile(Action): + def __init__(self:Self, + condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool, + actions:Sequence[Action] + ) -> None: + super().__init__() + self.condition:Condition|Callable[[dict[str, Any|None]], bool]|Selector|bool = condition + self.actions:Sequence[Action] = actions + +class Download(Action): + def __init__(self:Self, + url:str, + key:Optional[str] = None, + attribute:Optional[str] = None, + path:Optional[str] = None, + multiple:bool = False, + overwrite:bool = False + ) -> None: + super().__init__() + self.url:str = url + self.key:str|None = key + self.attribute:str|None = attribute + self.path:str|None = path + self.multiple:bool = multiple + self.overwrite:bool = overwrite \ No newline at end of file diff --git a/Python/AnP/Drivers/MySQLDriver.py b/Python/AnP/Drivers/MySQLDriver.py new file mode 100644 index 0000000..1f776fc --- /dev/null +++ b/Python/AnP/Drivers/MySQLDriver.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- \ No newline at end of file diff --git a/Python/AnP/Drivers/SQLServerDriver.py b/Python/AnP/Drivers/SQLServerDriver.py new file mode 100644 index 0000000..1f776fc --- /dev/null +++ b/Python/AnP/Drivers/SQLServerDriver.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- \ No newline at end of file diff --git a/Python/AnP/Drivers/SQLiteDriver.py b/Python/AnP/Drivers/SQLiteDriver.py new file mode 100644 index 0000000..1f776fc --- /dev/null +++ b/Python/AnP/Drivers/SQLiteDriver.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- \ No newline at end of file diff --git a/Python/AnP/Drivers/SeleniumDriver.py b/Python/AnP/Drivers/SeleniumDriver.py new file mode 100644 index 0000000..c704b1c --- /dev/null +++ b/Python/AnP/Drivers/SeleniumDriver.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from typing import Self +from AnP.DSLs.ScrapDSL import Action +from AnP.Interfaces.Application import AnPInterface +from AnP.Abstracts.BrowserAbstract import BrowserAbstract + +class SeleniumDriver(BrowserAbstract): + + def __init__(self:Self, anp:AnPInterface) -> None: + super().__init__(anp) \ No newline at end of file diff --git a/Python/AnP/Drivers/XMLScrapDriver.py b/Python/AnP/Drivers/XMLScrapDriver.py new file mode 100644 index 0000000..25af6b8 --- /dev/null +++ b/Python/AnP/Drivers/XMLScrapDriver.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from typing import Self, Any, Optional, Sequence, Callable +from requests import get, Response +from parsel import Selector as ParseSelector, SelectorList as ParseSelectorList +from AnP.DSLs.ScrapDSL import ( + Execute, Selector, CSS, XPath, Name, LinkText, Class, ID, Open, Close, Switch, + Go, Get, ForEach, Do, Condition, And, Or, Not, If, While, DoWhile, Download, + Action +) +from AnP.Interfaces.Application import AnPInterface +from AnP.Abstracts.BrowserAbstract import BrowserAbstract +from AnP.Models.XMLScrapTabModel import XMLScrapTabModel + +class XMLScrapDriver(BrowserAbstract): + + def __init__(self:Self, anp:AnPInterface, key:str, inputs:Optional[dict[str, Any|None]|Sequence[Any|None]] = None) -> None: + super().__init__(anp, key, XMLScrapTabModel, inputs) + + def run(self:Self, actions:Sequence[Action], shared:dict[str, Any|None] = {}) -> dict[str, Any|None]: + + self.do(Do(*actions), shared) + + return shared + + def __load(self:Self, url:str) -> Exception|None: + try: + + response:Response + + with get(url) as response: + self.tabs[self.tab_selected].set(response.text) + + except Exception as exception: + return exception + return None + + def execute(self:Self, action:Execute, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None: + try: + action.callback(shared) + action.done = True + except Exception as exception: + action.exception = exception + + def open(self:Self, action:Open, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None: + + url:str = shared[action.url] if action.url in shared and isinstance(shared[action.url], str) else action.url + + self.tabs[action.key] = XMLScrapTabModel(action.key, url) + self.tab_selected = action.key + if url is not None: + action.exception = self.__load(url) + action.done = action.exception is None + + def close(self:Self, action:Close, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None: + if action.key in self.tabs: + del self.tabs[action.key] + if len(self.tabs): + self.tab_selected = list(self.tabs.keys())[-1] + action.done = True + + def switch(self:Self, action:Switch, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None: + if action.key in self.tabs: + self.tab_selected = action.key + action.done = True + + def go(self:Self, action:Go, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None: + + url:str = shared[action.url] if action.url in shared and isinstance(shared[action.url], str) else action.url + + action.exception = self.__load(url) + action.done = action.exception is None + + def get_selector(self:Self, selector:str, from_item:Optional[Selector] = None, force_multiple:bool = False) -> ParseSelectorList|ParseSelector|None: + if self.tab_selected in self.tabs: + + path:ParseSelector = self.tabs[self.tab_selected].selector + items:ParseSelectorList = ( + path.css(selector) if isinstance(selector, CSS) else + path.xpath(selector) if isinstance(selector, XPath) else + path.css(f"[name='{selector}']") if isinstance(selector, Name) else + path.css(f"a:contains('{selector}')") if isinstance(selector, LinkText) else + path.css(f".{selector}") if isinstance(selector, Class) else + path.css(f"#{selector}") if isinstance(selector, ID) else + path.css(selector)) + + return ( + items if force_multiple or selector.multiple else + items[0] if len(items) else + None) + return [] if force_multiple or selector.multiple else None + + def get(self:Self, action:Get, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None: + + item:ParseSelector|ParseSelectorList|None = last_item if action.selector is None else self.get_selector(action.selector, last_item) + multiple:bool = False if action.selector is None else action.selector.multiple + attribute:str = "::text" if action.attribute == "CDATA" else f"::attr({action.attribute})" + + if action.reset and action.key in shared: + del shared[action.key] + + if multiple: + if action.key not in shared or not isinstance(shared[action.key], list): + shared[action.key] = [] + shared[action.key].extend([subitem.css(attribute).get() for subitem in item]) + else: + shared[action.key] = None if item is None else item.css(attribute).get() + + def foreach(self:Self, action:ForEach, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None: + + item:ParseSelector|Any|None + done:bool = True + + for item in ( + self.get_selector(action.items, last_item, True) if isinstance(action.items, Selector) else + action.items): + + if not self.anp.working(): + done = False + break + + if action.callback is not None: + try: + item = action.callback(item) + except Exception as exception: + action.exception = exception + done = False + break + + subactions:Do = Do(*action.actions) + + self.do(subactions, shared, item if isinstance(item, ParseSelector) else last_item) + if not subactions.done or subactions.exception is not None: + action.exception = subactions.exception + done = False + break + + action.done = done + + def _and(self:Self, action:And, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> bool: + + condition:Callable[[dict[str, Any|None]], bool]|Selector|Condition + + action.ok = True + + for condition in action.conditions: + if isinstance(condition, Condition): + self.do(condition, shared, last_item) + if not condition.ok: + action.ok = False + break + elif isinstance(condition, Selector): + if not self.get_selector(condition, last_item): + action.ok = False + break + elif callable(condition): + try: + if not condition(shared): + action.ok = False + break + except Exception as exception: + action.exception = exception + action.ok = False + break + + def _or(self:Self, action:Or, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> bool: + + condition:Callable[[dict[str, Any|None]], bool]|Selector|Condition + + action.ok = False + + for condition in action.conditions: + if isinstance(condition, Condition): + self.do(condition, shared, last_item) + if condition.ok: + action.ok = True + break + elif isinstance(condition, Selector): + if self.get_selector(condition, last_item): + action.ok = True + break + elif callable(condition): + try: + if condition(shared): + action.ok = True + break + except Exception as exception: + action.exception = exception + action.ok = False + break + + def _not(self:Self, action:Not, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> bool: + if isinstance(action.condition, Condition): + self.do(action.condition, shared, last_item) + action.ok = not action.condition.ok + elif isinstance(action.condition, Selector): + action.ok = not self.get_selector(action.condition, last_item) + elif callable(action.condition): + try: + action.ok = not action.condition(shared) + except Exception as exception: + action.exception = exception + action.ok = False + + def _if(self:Self, action:If, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None: + + if not isinstance(action.condition, Condition): + action.condition = Condition((action.condition,)) + + self.do(action.condition, shared, last_item) + + if action.condition.done and action.condition.exception is None: + + subactions:Do = Do(*((action.if_actions if action.condition.ok else action.else_actions) or [])) + + self.do(subactions, shared, last_item) + action.done = subactions.done and subactions.exception is None + action.exception = subactions.exception + + else: + action.done = action.condition.done + action.exception = action.condition.exception + + def _while(self:Self, action:While, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None: + + if not isinstance(action.condition, Condition): + action.condition = Condition((action.condition,)) + + while True: + if not self.anp.working(): + action.done = False + break + + self.do(action.condition, shared, last_item) + + if not action.condition.done or action.condition.exception is not None: + action.done = action.condition.done + action.exception = action.condition.exception + break + + if action.condition.ok: + + subactions:Do = Do(*action.actions) + + self.do(subactions, shared, last_item) + if not subactions.done or subactions.exception is not None: + action.exception = subactions.exception + action.done = False + break + + else: + action.done = True + break + + def do_while(self:Self, action:DoWhile, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None: + + if not isinstance(action.condition, Condition): + action.condition = Condition((action.condition,)) + + while True: + if not self.anp.working(): + action.done = False + break + + subactions:Do = Do(*action.actions) + + self.do(subactions, shared, last_item) + if not subactions.done or subactions.exception is not None: + action.exception = subactions.exception + action.done = False + break + + self.do(action.condition, shared, last_item) + + if not action.condition.done or action.condition.exception is not None: + action.done = action.condition.done + action.exception = action.condition.exception + break + + if not action.condition.ok: + action.done = True + break + + def download(self:Self, action:Download, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None: + if action.key or action.path: + try: + + response:Response + url:str = shared[action.url] if action.url in shared and isinstance(shared[action.url], str) else action.url + + with get(url) as response: + if response.content is not None: + if action.key: + if action.multiple: + if action.key not in shared or not isinstance(shared[action.key], list): + shared[action.key] = [] + shared[action.key].append(response.content) + action.done = True + elif action.overwrite or action.key not in shared: + shared[action.key] = response.content + action.done = True + else: + if action.overwrite or not self.anp.files.exists(action.path): + self.anp.files.save(action.path, response.content) + action.done = True + + except Exception as exception: + action.exception = exception \ No newline at end of file diff --git a/Python/AnP/Interfaces/Abstracts/BrowserAbstractInterface.py b/Python/AnP/Interfaces/Abstracts/BrowserAbstractInterface.py new file mode 100644 index 0000000..8333ba5 --- /dev/null +++ b/Python/AnP/Interfaces/Abstracts/BrowserAbstractInterface.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from typing import Optional, Self, Any, Sequence +from abc import ABC, abstractmethod +from AnP.DSLs.ScrapDSL import ( + Execute, Action, Open, Close, Switch, Wait, Do, Go, Get, ForEach, And, Or, Not, + If, While, DoWhile, Download +) +from AnP.Abstracts.ModelAbstract import ModelAbstract + +class BrowserAbstractInterface(ABC, ModelAbstract): + + @abstractmethod + def start(self:Self) -> None:pass + + @abstractmethod + def close(self:Self) -> None:pass + + @abstractmethod + def is_working(self:Self) -> bool:pass + + @abstractmethod + def run(self:Self, actions:Sequence[Action], shared:dict[str, Any|None] = {}) -> dict[str, Any|None]:pass + + @abstractmethod + def execute(self:Self, action:Execute, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def open(self:Self, action:Open, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def close(self:Self, action:Close, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def switch(self:Self, action:Switch, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def wait(self:Self, action:Wait, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def do(self:Self, action:Do, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def go(self:Self, action:Go, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def get(self:Self, action:Get, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def foreach(self:Self, action:ForEach, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def _and(self:Self, action:And, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> bool:pass + + @abstractmethod + def _or(self:Self, action:Or, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> bool:pass + + @abstractmethod + def _not(self:Self, action:Not, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> bool:pass + + @abstractmethod + def _if(self:Self, action:If, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def _while(self:Self, action:While, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def do_while(self:Self, action:DoWhile, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass + + @abstractmethod + def download(self:Self, action:Download, shared:dict[str, Any|None] = {}, last_item:Optional[Any] = None) -> None:pass \ No newline at end of file diff --git a/Python/AnP/Interfaces/Abstracts/FilesAbstractInterface.py b/Python/AnP/Interfaces/Abstracts/FilesAbstractInterface.py index 30ab2b2..0cb37a5 100644 --- a/Python/AnP/Interfaces/Abstracts/FilesAbstractInterface.py +++ b/Python/AnP/Interfaces/Abstracts/FilesAbstractInterface.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from typing import Self, Sequence +from typing import Self, Sequence, Any from abc import ABC, abstractmethod +from AnP.Abstracts.ModelAbstract import ModelAbstract -class FilesAbstractInterface(ABC): +class FilesAbstractInterface(ABC, ModelAbstract): @abstractmethod def get_parents(path:str, levels:int = 1) -> list[str]:pass @@ -43,4 +44,10 @@ class FilesAbstractInterface(ABC): def save(self:Self, path:str, content:str|bytes) -> bool:pass @abstractmethod - def delete(self:Self, path:str) -> bool:pass \ No newline at end of file + def delete(self:Self, path:str) -> bool:pass + + @abstractmethod + def load_json(self:Self, inputs:Any|None, only_dictionaries:bool = True) -> list[dict[str, Any|None]|Sequence[Any|None]]:pass + + @abstractmethod + def list_directory_items(self:Self, path:str) -> list[str]:pass \ No newline at end of file diff --git a/Python/AnP/Interfaces/Abstracts/HTTPServersAbstractInterface.py b/Python/AnP/Interfaces/Abstracts/HTTPServersAbstractInterface.py index 0748213..f709e49 100644 --- a/Python/AnP/Interfaces/Abstracts/HTTPServersAbstractInterface.py +++ b/Python/AnP/Interfaces/Abstracts/HTTPServersAbstractInterface.py @@ -3,8 +3,9 @@ from abc import ABC, abstractmethod from typing import Self +from AnP.Abstracts.ModelAbstract import ModelAbstract -class HTTPServersAbstractInterface(ABC): +class HTTPServersAbstractInterface(ABC, ModelAbstract): @abstractmethod def start(self:Self) -> None:pass diff --git a/Python/AnP/Interfaces/Abstracts/WebSocketServersAbstractInterface.py b/Python/AnP/Interfaces/Abstracts/WebSocketServersAbstractInterface.py index a6af92f..80b014e 100644 --- a/Python/AnP/Interfaces/Abstracts/WebSocketServersAbstractInterface.py +++ b/Python/AnP/Interfaces/Abstracts/WebSocketServersAbstractInterface.py @@ -3,8 +3,9 @@ from typing import Any, Self, Optional, Sequence from abc import ABC, abstractmethod +from AnP.Abstracts.ModelAbstract import ModelAbstract -class WebSocketServersAbstractInterface(ABC): +class WebSocketServersAbstractInterface(ABC, ModelAbstract): @abstractmethod def start(self:Self) -> None:pass diff --git a/Python/AnP/Interfaces/Application/AnPInterface.py b/Python/AnP/Interfaces/Application/AnPInterface.py index ab835d8..ba41fa6 100644 --- a/Python/AnP/Interfaces/Application/AnPInterface.py +++ b/Python/AnP/Interfaces/Application/AnPInterface.py @@ -20,6 +20,7 @@ from AnP.Interfaces.Managers.WebSocketServersManagerInterface import WebSocketSe from AnP.Interfaces.Managers.HTTPServersManagerInterface import HTTPServersManagerInterface from AnP.Interfaces.Managers.PseudoLoRAsManagerInterface import PseudoLoRAsManagerInterface from AnP.Interfaces.Managers.AIInterpretersManagerInterface import AIInterpretersManagerInterface +from AnP.Interfaces.Managers.BrowsersManagerInterface import BrowsersManagerInterface class AnPInterface(ABC): @@ -41,6 +42,7 @@ class AnPInterface(ABC): self.http_servers:HTTPServersManagerInterface = None self.pseudoloras:PseudoLoRAsManagerInterface = None self.ai_interpreters:AIInterpretersManagerInterface = None + self.browsers:BrowsersManagerInterface = None @abstractmethod def update(self:Self) -> None:pass diff --git a/Python/AnP/Interfaces/Managers/BrowsersManagerInterface.py b/Python/AnP/Interfaces/Managers/BrowsersManagerInterface.py new file mode 100644 index 0000000..8b5e258 --- /dev/null +++ b/Python/AnP/Interfaces/Managers/BrowsersManagerInterface.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from typing import Self, Any, Sequence +from abc import ABC, abstractmethod +from AnP.Interfaces.Abstracts.BrowserAbstractInterface import BrowserAbstractInterface +from AnP.DSLs.ScrapDSL import Action + +class BrowsersManagerInterface(ABC): + + @abstractmethod + def update(self:Self) -> None:pass + + @abstractmethod + def reset(self:Self) -> None:pass + + @abstractmethod + def close(self:Self) -> None:pass + + @abstractmethod + def add(self:Self, inputs:Any|None, overwrite:bool = False) -> None:pass + + @abstractmethod + def get(self:Self, key:str) -> BrowserAbstractInterface|None:pass + + @abstractmethod + def run(self:Self, key:str, actions:Sequence[Action], shared:dict[str, Any|None] = {}) -> dict[str, Any|None]:pass \ No newline at end of file diff --git a/Python/AnP/Managers/BrowsersManager.py b/Python/AnP/Managers/BrowsersManager.py new file mode 100644 index 0000000..072991c --- /dev/null +++ b/Python/AnP/Managers/BrowsersManager.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from typing import Self, Any, Sequence +from AnP.Interfaces.Application.AnPInterface import AnPInterface +from AnP.Abstracts.BrowserAbstract import BrowserAbstract +from AnP.Utils.Checks import Check +from AnP.DSLs.ScrapDSL import Action + +class BrowsersManager: + + def __init__(self:Self, anp:AnPInterface) -> None: + self.anp:AnPInterface = anp + self.__browsers:dict[str, BrowserAbstract] = {} + + self.update() + + def update(self:Self) -> None: + + key:str + + for key in ("default_browsers_files", "browsers_files", "default_browsers", "browsers"): + self.add(self.anp.settings.get(key), True) + + def reset(self:Self) -> None: + + self.__browsers = {} + + self.update() + + def close(self:Self) -> None: + for key, browser in list(self.__browsers.items()): + browser.close() + del self.__browsers[key] + + def add(self:Self, inputs:Any|None, overwrite:bool = False) -> None: + + subinputs:dict[str, Any|None] + + for subinputs in self.anp.files.load_json(inputs, True): + + key:str + browser:Any|None + + for key, browser in subinputs.items(): + + BrowserClass:type[BrowserAbstract]|None + + if Check.is_string(browser): + BrowserClass = self.anp.models.get(BrowserAbstract, browser) + elif issubclass(browser, BrowserAbstract): + BrowserClass = browser + elif Check.is_dictionary(browser) and "type" in browser and Check.is_string(browser["type"]): + BrowserClass = self.anp.models.get(BrowserAbstract, browser["type"]) + else: + continue + + if BrowserClass and ( + overwrite or key not in self.__browsers + ): + self.__browsers[key] = BrowserClass(self.anp, key, ( + browser if Check.is_dictionary(browser) else + None)) + + def get(self:Self, key:str) -> BrowserAbstract|None: + return self.__browsers.get(key, None) + + def run(self:Self, key:str, actions:Sequence[Action], shared:dict[str, Any|None] = {}) -> dict[str, Any|None]: + return self.__browsers[key].run(actions, shared) if key in self.__browsers else shared \ No newline at end of file diff --git a/Python/AnP/Managers/ControllersManager.py b/Python/AnP/Managers/ControllersManager.py index 8b2fe39..3532507 100644 --- a/Python/AnP/Managers/ControllersManager.py +++ b/Python/AnP/Managers/ControllersManager.py @@ -34,6 +34,10 @@ class ControllersManager: subinputs:dict[str, Any|None] for subinputs in self.anp.files.load_json(inputs, True): + + key:str + controller:Any|None + for key, controller in subinputs.items(): if Check.is_mark_key(key) and controller is None: continue diff --git a/Python/AnP/Managers/DispatchesManager.py b/Python/AnP/Managers/DispatchesManager.py index 6aef946..58cb3e4 100644 --- a/Python/AnP/Managers/DispatchesManager.py +++ b/Python/AnP/Managers/DispatchesManager.py @@ -33,6 +33,10 @@ class DispatchesManager: subinputs:dict[str, Any|None] for subinputs in self.anp.files.load_json(inputs, True): + + key:str + dispatch:Any|None + for key, dispatch in subinputs.items(): if Check.is_mark_key(key) and dispatch is None: continue diff --git a/Python/AnP/Models/CoordenateModel.py b/Python/AnP/Models/CoordenateModel.py new file mode 100644 index 0000000..178855f --- /dev/null +++ b/Python/AnP/Models/CoordenateModel.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from typing import Self + +class CoordenateModel: + + def __init__(self:Self, + x:float|int, + y:float|int + ) -> None: + self.x:float|int = x + self.y:float|int = y \ No newline at end of file diff --git a/Python/AnP/Models/PositionModel.py b/Python/AnP/Models/PositionModel.py new file mode 100644 index 0000000..54f7952 --- /dev/null +++ b/Python/AnP/Models/PositionModel.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from typing import Self +from random import random +from AnP.Models.CoordenateModel import CoordenateModel + +class PositionModel(CoordenateModel): + + def __init__(self:Self, + x:float|int, + y:float|int, + size:int|float|list[int|float, int|float]|tuple[int|float, int|float] = 0, + margin:int|float|list[int|float, int|float]|tuple[int|float, int|float] = 0 + ) -> None: + super().__init__(x, y) + self.size:CoordenateModel = CoordenateModel(*( + size if isinstance(size, (list, tuple)) else + (size, size) if isinstance(size, (int, float)) else + (0, 0))) + self.margin:CoordenateModel = CoordenateModel(*( + margin if isinstance(margin, (list, tuple)) else + (margin, margin) if isinstance(margin, (int, float)) else + (0, 0))) + + def get(self:Self) -> tuple[float, float]: + return ( + self.x + random() * (self.size.x - 2 * self.margin.x) + self.margin.x, + self.y + random() * (self.size.y - 2 * self.margin.y) + self.margin.y + ) \ No newline at end of file diff --git a/Python/AnP/Models/XMLScrapTabModel.py b/Python/AnP/Models/XMLScrapTabModel.py new file mode 100644 index 0000000..e294ddc --- /dev/null +++ b/Python/AnP/Models/XMLScrapTabModel.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from typing import Self +from parsel import Selector + +class XMLScrapTabModel: + + def __init__(self:Self, key:str, url:str) -> None: + self.key:str = key + self.url:str = url + self.data:str = "" + self.selector:Selector|None = None + + def set(self:Self, data:str) -> None: + self.data = data + self.selector = Selector(text = data, type = "xml") \ No newline at end of file diff --git a/Python/AnP/Utils/Common.py b/Python/AnP/Utils/Common.py index 8e5ad60..6929322 100644 --- a/Python/AnP/Utils/Common.py +++ b/Python/AnP/Utils/Common.py @@ -10,7 +10,7 @@ from mimetypes import guess_type as get_mime_by_extension from inspect import FrameInfo, stack as get_stack from json import dumps as json_encode, loads as json_decode from base64 import b64encode as base64_encode, b64decode as base64_decode -from random import choice as random_choice +from random import choice as random_choice, random as random_number from AnP.Utils.Checks import Check from AnP.Utils.Patterns import RE @@ -350,4 +350,12 @@ class Common: def random(items:str|Sequence[Any|None]) -> Any|None: return random_choice(items) if ( Check.is_string(items) or Check.is_array(items) - ) and len(items) else None \ No newline at end of file + ) and len(items) else None + + @staticmethod + def random_number(minimum:Optional[int|float] = None, maximum:Optional[int|float] = None) -> int|float: + if minimum is None: + return random_number() + if maximum is None: + return random_number() * minimum + return random_number() * (maximum - minimum) + minimum \ No newline at end of file diff --git a/Python/base.py b/Python/base.py index 8337c0c..f892bc9 100644 --- a/Python/base.py +++ b/Python/base.py @@ -8,6 +8,7 @@ from AnP.Drivers.HTTPSocketServerDriver import HTTPSocketServerDriver from AnP.Drivers.HTTPServerDriver import HTTPServerDriver from AnP.Drivers.OllamaDriver import OllamaDriver from AnP.Drivers.FilesDriver import FilesDriver +from AnP.Drivers.XMLScrapDriver import XMLScrapDriver from AnP.Controllers.CloneController import CloneController from AnP.Controllers.AITranslateController import AITranslateController @@ -20,7 +21,8 @@ inputs:dict[str, dict[str, Any|None]] = { "HTTPSocketServerDriver" : HTTPSocketServerDriver, "HTTPServerDriver" : HTTPServerDriver, "OllamaDriver" : OllamaDriver, - "FilesDriver" : FilesDriver + "FilesDriver" : FilesDriver, + "XMLScrapDriver" : XMLScrapDriver } }