first commit
This commit is contained in:
+94
@@ -0,0 +1,94 @@
|
||||
<?php
|
||||
/**
|
||||
* Created by PhpStorm.
|
||||
* User: Jaeger <JaegerCode@gmail.com>
|
||||
* Date: 2017/9/22
|
||||
*/
|
||||
|
||||
namespace QL;
|
||||
use Closure;
|
||||
use Tightenco\Collect\Support\Collection;
|
||||
|
||||
class Config
|
||||
{
|
||||
protected static $instance = null;
|
||||
|
||||
protected $plugins;
|
||||
protected $binds;
|
||||
|
||||
/**
|
||||
* Config constructor.
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this->plugins = new Collection();
|
||||
$this->binds = new Collection();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Get the Config instance
|
||||
*
|
||||
* @return null|Config
|
||||
*/
|
||||
public static function getInstance()
|
||||
{
|
||||
self::$instance || self::$instance = new self();
|
||||
return self::$instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Global installation plugin
|
||||
*
|
||||
* @param $plugins
|
||||
* @param array ...$opt
|
||||
* @return $this
|
||||
*/
|
||||
public function use($plugins,...$opt)
|
||||
{
|
||||
if(is_string($plugins)){
|
||||
$this->plugins->push([$plugins,$opt]);
|
||||
}else{
|
||||
$this->plugins = $this->plugins->merge($plugins);
|
||||
}
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Global binding custom method
|
||||
*
|
||||
* @param string $name
|
||||
* @param Closure $provider
|
||||
* @return $this
|
||||
*/
|
||||
public function bind(string $name, Closure $provider)
|
||||
{
|
||||
$this->binds[$name] = $provider;
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function bootstrap(QueryList $queryList)
|
||||
{
|
||||
$this->installPlugins($queryList);
|
||||
$this->installBind($queryList);
|
||||
}
|
||||
|
||||
protected function installPlugins(QueryList $queryList)
|
||||
{
|
||||
$this->plugins->each(function($plugin) use($queryList){
|
||||
if(is_string($plugin)){
|
||||
$queryList->use($plugin);
|
||||
}else{
|
||||
$queryList->use($plugin[0],...$plugin[1]);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
protected function installBind(QueryList $queryList)
|
||||
{
|
||||
$this->binds->each(function ($provider,$name) use($queryList){
|
||||
$queryList->bind($name,$provider);
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
<?php
|
||||
/**
|
||||
* Created by PhpStorm.
|
||||
* User: Jaeger <JaegerCode@gmail.com>
|
||||
* Date: 2017/9/22
|
||||
*/
|
||||
|
||||
namespace QL\Contracts;
|
||||
|
||||
use QL\QueryList;
|
||||
|
||||
interface PluginContract
|
||||
{
|
||||
public static function install(QueryList $queryList,...$opt);
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
<?php
|
||||
/**
|
||||
* Created by PhpStorm.
|
||||
* User: Jaeger <JaegerCode@gmail.com>
|
||||
* Date: 2017/9/20
|
||||
*/
|
||||
|
||||
namespace QL\Contracts;
|
||||
|
||||
use QL\Kernel;
|
||||
|
||||
interface ServiceProviderContract
|
||||
{
|
||||
public function register(Kernel $kernel);
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
<?php
|
||||
/**
|
||||
* Created by PhpStorm.
|
||||
* User: Jaeger <JaegerCode@gmail.com>
|
||||
* Date: 2017/9/19
|
||||
*/
|
||||
|
||||
namespace QL\Dom;
|
||||
|
||||
use phpQueryObject;
|
||||
|
||||
class Dom
|
||||
{
|
||||
|
||||
protected $document;
|
||||
|
||||
/**
|
||||
* Dom constructor.
|
||||
*/
|
||||
public function __construct(phpQueryObject $document)
|
||||
{
|
||||
$this->document = $document;
|
||||
}
|
||||
|
||||
public function find($selector)
|
||||
{
|
||||
$elements = $this->document->find($selector);
|
||||
return new Elements($elements);
|
||||
}
|
||||
}
|
||||
+260
@@ -0,0 +1,260 @@
|
||||
<?php
|
||||
/**
|
||||
* Created by PhpStorm.
|
||||
* User: Jaeger <JaegerCode@gmail.com>
|
||||
* Date: 2017/9/19
|
||||
*/
|
||||
|
||||
namespace QL\Dom;
|
||||
|
||||
use phpDocumentor\Reflection\Types\Null_;
|
||||
use phpQueryObject;
|
||||
use Tightenco\Collect\Support\Collection;
|
||||
|
||||
/**
|
||||
* Class Elements
|
||||
* @package QL\Dom
|
||||
*
|
||||
* @method Elements toReference(&$var)
|
||||
* @method Elements documentFragment($state = null)
|
||||
* @method Elements toRoot()
|
||||
* @method Elements getDocumentIDRef(&$documentID)
|
||||
* @method Elements getDocument()
|
||||
* @method \DOMDocument getDOMDocument()
|
||||
* @method Elements getDocumentID()
|
||||
* @method Elements unloadDocument()
|
||||
* @method bool isHTML()
|
||||
* @method bool isXHTML()
|
||||
* @method bool isXML()
|
||||
* @method string serialize()
|
||||
* @method array serializeArray($submit = null)
|
||||
* @method \DOMElement|\DOMElement[] get($index = null, $callback1 = null, $callback2 = null, $callback3 = null)
|
||||
* @method string|array getString($index = null, $callback1 = null, $callback2 = null, $callback3 = null)
|
||||
* @method string|array getStrings($index = null, $callback1 = null, $callback2 = null, $callback3 = null)
|
||||
* @method Elements newInstance($newStack = null)
|
||||
* @method Elements find($selectors, $context = null, $noHistory = false)
|
||||
* @method Elements|bool is($selector, $nodes = null)
|
||||
* @method Elements filterCallback($callback, $_skipHistory = false)
|
||||
* @method Elements filter($selectors, $_skipHistory = false)
|
||||
* @method Elements load($url, $data = null, $callback = null)
|
||||
* @method Elements trigger($type, $data = [])
|
||||
* @method Elements triggerHandler($type, $data = [])
|
||||
* @method Elements bind($type, $data, $callback = null)
|
||||
* @method Elements unbind($type = null, $callback = null)
|
||||
* @method Elements change($callback = null)
|
||||
* @method Elements submit($callback = null)
|
||||
* @method Elements click($callback = null)
|
||||
* @method Elements wrapAllOld($wrapper)
|
||||
* @method Elements wrapAll($wrapper)
|
||||
* @method Elements wrapAllPHP($codeBefore, $codeAfter)
|
||||
* @method Elements wrap($wrapper)
|
||||
* @method Elements wrapPHP($codeBefore, $codeAfter)
|
||||
* @method Elements wrapInner($wrapper)
|
||||
* @method Elements wrapInnerPHP($codeBefore, $codeAfter)
|
||||
* @method Elements contents()
|
||||
* @method Elements contentsUnwrap()
|
||||
* @method Elements switchWith($markup)
|
||||
* @method Elements eq($num)
|
||||
* @method Elements size()
|
||||
* @method Elements length()
|
||||
* @method int count()
|
||||
* @method Elements end($level = 1)
|
||||
* @method Elements _clone()
|
||||
* @method Elements replaceWithPHP($code)
|
||||
* @method Elements replaceWith($content)
|
||||
* @method Elements replaceAll($selector)
|
||||
* @method Elements remove($selector = null)
|
||||
* @method Elements|string markup($markup = null, $callback1 = null, $callback2 = null, $callback3 = null)
|
||||
* @method string markupOuter($callback1 = null, $callback2 = null, $callback3 = null)
|
||||
* @method Elements|string html($html = null, $callback1 = null, $callback2 = null, $callback3 = null)
|
||||
* @method Elements|string xml($xml = null, $callback1 = null, $callback2 = null, $callback3 = null)
|
||||
* @method string htmlOuter($callback1 = null, $callback2 = null, $callback3 = null)
|
||||
* @method string xmlOuter($callback1 = null, $callback2 = null, $callback3 = null)
|
||||
* @method Elements php($code)
|
||||
* @method string markupPHP($code)
|
||||
* @method string markupOuterPHP()
|
||||
* @method Elements children($selector)
|
||||
* @method Elements ancestors($selector)
|
||||
* @method Elements append($content)
|
||||
* @method Elements appendPHP($content)
|
||||
* @method Elements appendTo($seletor)
|
||||
* @method Elements prepend($content)
|
||||
* @method Elements prependPHP($content)
|
||||
* @method Elements prependTo($seletor)
|
||||
* @method Elements before($content)
|
||||
* @method Elements beforePHP($content)
|
||||
* @method Elements insertBefore($seletor)
|
||||
* @method Elements after($content)
|
||||
* @method Elements afterPHP($content)
|
||||
* @method Elements insertAfter($seletor)
|
||||
* @method Elements insert($target, $type)
|
||||
* @method int index($subject)
|
||||
* @method Elements slice($start, $end = null)
|
||||
* @method Elements reverse()
|
||||
* @method Elements|string text($text = null, $callback1 = null, $callback2 = null, $callback3 = null)
|
||||
* @method Elements plugin($class, $file = null)
|
||||
* @method Elements _next($selector = null)
|
||||
* @method Elements _prev($selector = null)
|
||||
* @method Elements prev($selector = null)
|
||||
* @method Elements prevAll($selector = null)
|
||||
* @method Elements nextAll($selector = null)
|
||||
* @method Elements siblings($selector = null)
|
||||
* @method Elements not($selector = null)
|
||||
* @method Elements add($selector = null)
|
||||
* @method Elements parent($selector = null)
|
||||
* @method Elements parents($selector = null)
|
||||
* @method Elements stack($nodeTypes = null)
|
||||
* @method Elements|string attr($attr = null, $value = null)
|
||||
* @method Elements attrPHP($attr, $code)
|
||||
* @method Elements removeAttr($attr)
|
||||
* @method Elements|string val($val = null)
|
||||
* @method Elements andSelf()
|
||||
* @method Elements addClass($className)
|
||||
* @method Elements addClassPHP($className)
|
||||
* @method bool hasClass($className)
|
||||
* @method Elements removeClass($className)
|
||||
* @method Elements toggleClass($className)
|
||||
* @method Elements _empty()
|
||||
* @method Elements callback($callback, $param1 = null, $param2 = null, $param3 = null)
|
||||
* @method string data($key, $value = null)
|
||||
* @method Elements removeData($key)
|
||||
* @method void rewind()
|
||||
* @method Elements current()
|
||||
* @method int key()
|
||||
* @method Elements next($cssSelector = null)
|
||||
* @method bool valid()
|
||||
* @method bool offsetExists($offset)
|
||||
* @method Elements offsetGet($offset)
|
||||
* @method void offsetSet($offset, $value)
|
||||
* @method string whois($oneNode)
|
||||
* @method Elements dump()
|
||||
* @method Elements dumpWhois()
|
||||
* @method Elements dumpLength()
|
||||
* @method Elements dumpTree($html, $title)
|
||||
* @method dumpDie()
|
||||
*/
|
||||
class Elements
|
||||
{
|
||||
/**
|
||||
* @var phpQueryObject
|
||||
*/
|
||||
protected $elements;
|
||||
|
||||
/**
|
||||
* Elements constructor.
|
||||
* @param $elements
|
||||
*/
|
||||
public function __construct(phpQueryObject $elements)
|
||||
{
|
||||
$this->elements = $elements;
|
||||
}
|
||||
|
||||
public function __get($name)
|
||||
{
|
||||
return property_exists($this->elements, $name) ? $this->elements->$name : $this->elements->attr($name);
|
||||
}
|
||||
|
||||
public function __call($name, $arguments)
|
||||
{
|
||||
$obj = call_user_func_array([$this->elements, $name], $arguments);
|
||||
if ($obj instanceof phpQueryObject) {
|
||||
$obj = new self($obj);
|
||||
} else if (is_string($obj)) {
|
||||
$obj = trim($obj);
|
||||
}
|
||||
return $obj;
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterating elements
|
||||
*
|
||||
* @param callable $callback
|
||||
*
|
||||
* @return $this
|
||||
*/
|
||||
public function each(callable $callback)
|
||||
{
|
||||
foreach ($this->elements as $key => $element) {
|
||||
$break = $callback(new self(pq($element)), $key);
|
||||
if ($break === false) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterating elements
|
||||
*
|
||||
* @param $callback
|
||||
* @return \Illuminate\Support\Collection|\Tightenco\Collect\Support\Collection
|
||||
*/
|
||||
public function map($callback)
|
||||
{
|
||||
$collection = new Collection();
|
||||
$this->elements->each(function ($dom) use (& $collection, $callback) {
|
||||
$collection->push($callback(new self(pq($dom))));
|
||||
});
|
||||
return $collection;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the attributes of all the elements
|
||||
*
|
||||
* @param string $attr HTML attribute name
|
||||
* @return \Illuminate\Support\Collection|\Tightenco\Collect\Support\Collection
|
||||
*/
|
||||
public function attrs($attr)
|
||||
{
|
||||
return $this->map(function ($item) use ($attr) {
|
||||
return $item->attr($attr);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the text of all the elements
|
||||
*
|
||||
* @return \Illuminate\Support\Collection|\Tightenco\Collect\Support\Collection
|
||||
*/
|
||||
public function texts()
|
||||
{
|
||||
return $this->map(function ($item) {
|
||||
return trim($item->text());
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the html of all the elements
|
||||
*
|
||||
* @return \Illuminate\Support\Collection|\Tightenco\Collect\Support\Collection
|
||||
*/
|
||||
public function htmls()
|
||||
{
|
||||
return $this->map(function ($item) {
|
||||
return trim($item->html());
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the htmlOuter of all the elements
|
||||
*
|
||||
* @return \Illuminate\Support\Collection|\Tightenco\Collect\Support\Collection
|
||||
*/
|
||||
public function htmlOuters()
|
||||
{
|
||||
return $this->map(function ($item) {
|
||||
return trim($item->htmlOuter());
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return phpQueryObject
|
||||
*/
|
||||
public function getElements(): phpQueryObject
|
||||
{
|
||||
return $this->elements;
|
||||
}
|
||||
|
||||
}
|
||||
+322
@@ -0,0 +1,322 @@
|
||||
<?php
|
||||
/**
|
||||
* Created by PhpStorm.
|
||||
* User: Jaeger <JaegerCode@gmail.com>
|
||||
* Date: 2017/9/21
|
||||
*/
|
||||
|
||||
namespace QL\Dom;
|
||||
|
||||
use Tightenco\Collect\Support\Collection;
|
||||
use phpQuery;
|
||||
use phpQueryObject;
|
||||
use QL\QueryList;
|
||||
use Closure;
|
||||
|
||||
class Query
|
||||
{
|
||||
protected $html;
|
||||
/**
|
||||
* @var \phpQueryObject
|
||||
*/
|
||||
protected $document;
|
||||
protected $rules;
|
||||
protected $range = null;
|
||||
protected $ql;
|
||||
/**
|
||||
* @var Collection
|
||||
*/
|
||||
protected $data;
|
||||
|
||||
|
||||
public function __construct(QueryList $ql)
|
||||
{
|
||||
$this->ql = $ql;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param bool $rel
|
||||
* @return String
|
||||
*/
|
||||
public function getHtml($rel = true)
|
||||
{
|
||||
return $rel ? $this->document->htmlOuter() : $this->html;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param $html
|
||||
* @param null $charset
|
||||
* @return QueryList
|
||||
*/
|
||||
public function setHtml($html, $charset = null)
|
||||
{
|
||||
$this->html = value($html);
|
||||
$this->destroyDocument();
|
||||
$this->document = phpQuery::newDocumentHTML($this->html, $charset);
|
||||
return $this->ql;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get crawl results
|
||||
*
|
||||
* @param Closure|null $callback
|
||||
* @return Collection|static
|
||||
*/
|
||||
public function getData(Closure $callback = null)
|
||||
{
|
||||
return $this->handleData($this->data, $callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Collection $data
|
||||
*/
|
||||
public function setData(Collection $data)
|
||||
{
|
||||
$this->data = $data;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Searches for all elements that match the specified expression.
|
||||
*
|
||||
* @param $selector A string containing a selector expression to match elements against.
|
||||
* @return Elements
|
||||
*/
|
||||
public function find($selector)
|
||||
{
|
||||
return (new Dom($this->document))->find($selector);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set crawl rule
|
||||
*
|
||||
* $rules = [
|
||||
* 'rule_name1' => ['selector','HTML attribute | text | html','Tag filter list','callback'],
|
||||
* 'rule_name2' => ['selector','HTML attribute | text | html','Tag filter list','callback'],
|
||||
* // ...
|
||||
* ]
|
||||
*
|
||||
* @param array $rules
|
||||
* @return QueryList
|
||||
*/
|
||||
public function rules(array $rules)
|
||||
{
|
||||
$this->rules = $rules;
|
||||
return $this->ql;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Set the slice area for crawl list
|
||||
*
|
||||
* @param $selector
|
||||
* @return QueryList
|
||||
*/
|
||||
public function range($selector)
|
||||
{
|
||||
$this->range = $selector;
|
||||
return $this->ql;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove HTML head,try to solve the garbled
|
||||
*
|
||||
* @return QueryList
|
||||
*/
|
||||
public function removeHead()
|
||||
{
|
||||
$html = preg_replace('/(<head>|<head\s+.+?>).+<\/head>/is', '<head></head>', $this->html);
|
||||
$this->setHtml($html);
|
||||
return $this->ql;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the query rule
|
||||
*
|
||||
* @param Closure|null $callback
|
||||
* @return QueryList
|
||||
*/
|
||||
public function query(Closure $callback = null)
|
||||
{
|
||||
$this->data = $this->getList();
|
||||
$this->data = $this->handleData($this->data, $callback);
|
||||
return $this->ql;
|
||||
}
|
||||
|
||||
public function handleData(Collection $data, $callback)
|
||||
{
|
||||
if (is_callable($callback)) {
|
||||
if (empty($this->range)) {
|
||||
$data = new Collection($callback($data->all(), null));
|
||||
} else {
|
||||
$data = $data->map($callback);
|
||||
}
|
||||
}
|
||||
|
||||
return $data;
|
||||
}
|
||||
|
||||
protected function getList()
|
||||
{
|
||||
$data = [];
|
||||
if (empty($this->range)) {
|
||||
foreach ($this->rules as $key => $reg_value) {
|
||||
$rule = $this->parseRule($reg_value);
|
||||
$contentElements = $this->document->find($rule['selector']);
|
||||
$data[$key] = $this->extractContent($contentElements, $key, $rule);
|
||||
}
|
||||
} else {
|
||||
$rangeElements = $this->document->find($this->range);
|
||||
$i = 0;
|
||||
foreach ($rangeElements as $element) {
|
||||
foreach ($this->rules as $key => $reg_value) {
|
||||
$rule = $this->parseRule($reg_value);
|
||||
$contentElements = pq($element)->find($rule['selector']);
|
||||
$data[$i][$key] = $this->extractContent($contentElements, $key, $rule);
|
||||
}
|
||||
$i++;
|
||||
}
|
||||
}
|
||||
|
||||
return new Collection($data);
|
||||
}
|
||||
|
||||
protected function extractContent(phpQueryObject $pqObj, $ruleName, $rule)
|
||||
{
|
||||
switch ($rule['attr']) {
|
||||
case 'text':
|
||||
$content = $this->allowTags($pqObj->html(), $rule['filter_tags']);
|
||||
break;
|
||||
case 'texts':
|
||||
$content = (new Elements($pqObj))->map(function (Elements $element) use ($rule) {
|
||||
return $this->allowTags($element->html(), $rule['filter_tags']);
|
||||
})->all();
|
||||
break;
|
||||
case 'html':
|
||||
$content = $this->stripTags($pqObj->html(), $rule['filter_tags']);
|
||||
break;
|
||||
case 'htmls':
|
||||
$content = (new Elements($pqObj))->map(function (Elements $element) use ($rule) {
|
||||
return $this->stripTags($element->html(), $rule['filter_tags']);
|
||||
})->all();
|
||||
break;
|
||||
case 'htmlOuter':
|
||||
$content = $this->stripTags($pqObj->htmlOuter(), $rule['filter_tags']);
|
||||
break;
|
||||
case 'htmlOuters':
|
||||
$content = (new Elements($pqObj))->map(function (Elements $element) use ($rule) {
|
||||
return $this->stripTags($element->htmlOuter(), $rule['filter_tags']);
|
||||
})->all();
|
||||
break;
|
||||
default:
|
||||
if(preg_match('/attr\((.+)\)/', $rule['attr'], $arr)) {
|
||||
$content = $pqObj->attr($arr[1]);
|
||||
} elseif (preg_match('/attrs\((.+)\)/', $rule['attr'], $arr)) {
|
||||
$content = (new Elements($pqObj))->attrs($arr[1])->all();
|
||||
} else {
|
||||
$content = $pqObj->attr($rule['attr']);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
if (is_callable($rule['handle_callback'])) {
|
||||
$content = call_user_func($rule['handle_callback'], $content, $ruleName);
|
||||
}
|
||||
|
||||
return $content;
|
||||
}
|
||||
|
||||
protected function parseRule($rule)
|
||||
{
|
||||
$result = [];
|
||||
$result['selector'] = $rule[0];
|
||||
$result['attr'] = $rule[1];
|
||||
$result['filter_tags'] = $rule[2] ?? '';
|
||||
$result['handle_callback'] = $rule[3] ?? null;
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 去除特定的html标签
|
||||
* @param string $html
|
||||
* @param string $tags_str 多个标签名之间用空格隔开
|
||||
* @return string
|
||||
*/
|
||||
protected function stripTags($html, $tags_str)
|
||||
{
|
||||
$tagsArr = $this->tag($tags_str);
|
||||
$html = $this->removeTags($html, $tagsArr[1]);
|
||||
$p = array();
|
||||
foreach ($tagsArr[0] as $tag) {
|
||||
$p[] = "/(<(?:\/" . $tag . "|" . $tag . ")[^>]*>)/i";
|
||||
}
|
||||
$html = preg_replace($p, "", trim($html));
|
||||
return $html;
|
||||
}
|
||||
|
||||
/**
|
||||
* 保留特定的html标签
|
||||
* @param string $html
|
||||
* @param string $tags_str 多个标签名之间用空格隔开
|
||||
* @return string
|
||||
*/
|
||||
protected function allowTags($html, $tags_str)
|
||||
{
|
||||
$tagsArr = $this->tag($tags_str);
|
||||
$html = $this->removeTags($html, $tagsArr[1]);
|
||||
$allow = '';
|
||||
foreach ($tagsArr[0] as $tag) {
|
||||
$allow .= "<$tag> ";
|
||||
}
|
||||
return strip_tags(trim($html), $allow);
|
||||
}
|
||||
|
||||
protected function tag($tags_str)
|
||||
{
|
||||
$tagArr = preg_split("/\s+/", $tags_str, -1, PREG_SPLIT_NO_EMPTY);
|
||||
$tags = array(array(), array());
|
||||
foreach ($tagArr as $tag) {
|
||||
if (preg_match('/-(.+)/', $tag, $arr)) {
|
||||
array_push($tags[1], $arr[1]);
|
||||
} else {
|
||||
array_push($tags[0], $tag);
|
||||
}
|
||||
}
|
||||
return $tags;
|
||||
}
|
||||
|
||||
/**
|
||||
* 移除特定的html标签
|
||||
* @param string $html
|
||||
* @param array $tags 标签数组
|
||||
* @return string
|
||||
*/
|
||||
protected function removeTags($html, $tags)
|
||||
{
|
||||
$tag_str = '';
|
||||
if (count($tags)) {
|
||||
foreach ($tags as $tag) {
|
||||
$tag_str .= $tag_str ? ',' . $tag : $tag;
|
||||
}
|
||||
// phpQuery::$defaultCharset = $this->inputEncoding?$this->inputEncoding:$this->htmlEncoding;
|
||||
$doc = phpQuery::newDocumentHTML($html);
|
||||
pq($doc)->find($tag_str)->remove();
|
||||
$html = pq($doc)->htmlOuter();
|
||||
$doc->unloadDocument();
|
||||
}
|
||||
return $html;
|
||||
}
|
||||
|
||||
protected function destroyDocument()
|
||||
{
|
||||
if ($this->document instanceof phpQueryObject) {
|
||||
$this->document->unloadDocument();
|
||||
}
|
||||
}
|
||||
|
||||
public function __destruct()
|
||||
{
|
||||
$this->destroyDocument();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
<?php
|
||||
/**
|
||||
* Created by PhpStorm.
|
||||
* User: Jaeger <JaegerCode@gmail.com>
|
||||
* Date: 2017/9/21
|
||||
*/
|
||||
|
||||
namespace QL\Exceptions;
|
||||
|
||||
use Exception;
|
||||
|
||||
class ServiceNotFoundException extends Exception
|
||||
{
|
||||
|
||||
}
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
<?php
|
||||
/**
|
||||
* Created by PhpStorm.
|
||||
* User: Jaeger <JaegerCode@gmail.com>
|
||||
* Date: 2017/9/21
|
||||
*/
|
||||
|
||||
namespace QL;
|
||||
|
||||
use QL\Contracts\ServiceProviderContract;
|
||||
use QL\Exceptions\ServiceNotFoundException;
|
||||
use QL\Providers\EncodeServiceProvider;
|
||||
use Closure;
|
||||
use QL\Providers\HttpServiceProvider;
|
||||
use QL\Providers\PluginServiceProvider;
|
||||
use QL\Providers\SystemServiceProvider;
|
||||
use Tightenco\Collect\Support\Collection;
|
||||
|
||||
class Kernel
|
||||
{
|
||||
protected $providers = [
|
||||
SystemServiceProvider::class,
|
||||
HttpServiceProvider::class,
|
||||
EncodeServiceProvider::class,
|
||||
PluginServiceProvider::class
|
||||
];
|
||||
|
||||
protected $binds;
|
||||
protected $ql;
|
||||
|
||||
/**
|
||||
* Kernel constructor.
|
||||
* @param $ql
|
||||
*/
|
||||
public function __construct(QueryList $ql)
|
||||
{
|
||||
$this->ql = $ql;
|
||||
$this->binds = new Collection();
|
||||
}
|
||||
|
||||
public function bootstrap()
|
||||
{
|
||||
//注册服务提供者
|
||||
$this->registerProviders();
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function registerProviders()
|
||||
{
|
||||
foreach ($this->providers as $provider) {
|
||||
$this->register(new $provider());
|
||||
}
|
||||
}
|
||||
|
||||
public function bind(string $name,Closure $provider)
|
||||
{
|
||||
$this->binds[$name] = $provider;
|
||||
}
|
||||
|
||||
public function getService(string $name)
|
||||
{
|
||||
if(!$this->binds->offsetExists($name)){
|
||||
throw new ServiceNotFoundException("Service: {$name} not found!");
|
||||
}
|
||||
return $this->binds[$name];
|
||||
}
|
||||
|
||||
private function register(ServiceProviderContract $instance)
|
||||
{
|
||||
$instance->register($this);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
<?php
|
||||
/**
|
||||
* Created by PhpStorm.
|
||||
* User: Jaeger <JaegerCode@gmail.com>
|
||||
* Date: 2017/9/20
|
||||
*/
|
||||
|
||||
namespace QL\Providers;
|
||||
|
||||
use QL\Contracts\ServiceProviderContract;
|
||||
use QL\Kernel;
|
||||
use QL\Services\EncodeService;
|
||||
|
||||
class EncodeServiceProvider implements ServiceProviderContract
|
||||
{
|
||||
public function register(Kernel $kernel)
|
||||
{
|
||||
$kernel->bind('encoding',function (string $outputEncoding,string $inputEncoding = null){
|
||||
return EncodeService::convert($this,$outputEncoding,$inputEncoding);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
<?php
|
||||
/**
|
||||
* Created by PhpStorm.
|
||||
* User: Jaeger <JaegerCode@gmail.com>
|
||||
* Date: 2017/9/22
|
||||
*/
|
||||
|
||||
namespace QL\Providers;
|
||||
|
||||
|
||||
use QL\Contracts\ServiceProviderContract;
|
||||
use QL\Kernel;
|
||||
use QL\Services\HttpService;
|
||||
use QL\Services\MultiRequestService;
|
||||
|
||||
class HttpServiceProvider implements ServiceProviderContract
|
||||
{
|
||||
public function register(Kernel $kernel)
|
||||
{
|
||||
$kernel->bind('get',function (...$args){
|
||||
return HttpService::get($this,...$args);
|
||||
});
|
||||
|
||||
$kernel->bind('post',function (...$args){
|
||||
return HttpService::post($this,...$args);
|
||||
});
|
||||
|
||||
$kernel->bind('postJson',function (...$args){
|
||||
return HttpService::postJson($this,...$args);
|
||||
});
|
||||
|
||||
$kernel->bind('multiGet',function (...$args){
|
||||
return new MultiRequestService($this,'get',...$args);
|
||||
});
|
||||
|
||||
$kernel->bind('multiPost',function (...$args){
|
||||
return new MultiRequestService($this,'post',...$args);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
<?php
|
||||
/**
|
||||
* Created by PhpStorm.
|
||||
* User: Jaeger <JaegerCode@gmail.com>
|
||||
* Date: 2017/9/22
|
||||
*/
|
||||
|
||||
namespace QL\Providers;
|
||||
|
||||
use QL\Contracts\ServiceProviderContract;
|
||||
use QL\Kernel;
|
||||
use QL\Services\PluginService;
|
||||
|
||||
class PluginServiceProvider implements ServiceProviderContract
|
||||
{
|
||||
public function register(Kernel $kernel)
|
||||
{
|
||||
$kernel->bind('use',function ($plugins,...$opt){
|
||||
return PluginService::install($this,$plugins,...$opt);
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
<?php
|
||||
/**
|
||||
* Created by PhpStorm.
|
||||
* User: Jaeger <JaegerCode@gmail.com>
|
||||
* Date: 2017/9/22
|
||||
*/
|
||||
|
||||
namespace QL\Providers;
|
||||
|
||||
use QL\Contracts\ServiceProviderContract;
|
||||
use QL\Kernel;
|
||||
use Closure;
|
||||
|
||||
class SystemServiceProvider implements ServiceProviderContract
|
||||
{
|
||||
public function register(Kernel $kernel)
|
||||
{
|
||||
$kernel->bind('html',function (...$args){
|
||||
$this->setHtml(...$args);
|
||||
return $this;
|
||||
});
|
||||
|
||||
$kernel->bind('queryData',function (Closure $callback = null){
|
||||
return $this->query()->getData($callback)->all();
|
||||
});
|
||||
|
||||
$kernel->bind('pipe',function (Closure $callback = null){
|
||||
return $callback($this);
|
||||
});
|
||||
|
||||
}
|
||||
}
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
<?php
|
||||
/**
|
||||
* QueryList
|
||||
*
|
||||
* 一个基于phpQuery的通用列表采集类
|
||||
*
|
||||
* @author Jaeger
|
||||
* @email JaegerCode@gmail.com
|
||||
* @link https://github.com/jae-jae/QueryList
|
||||
* @version 4.0.0
|
||||
*
|
||||
*/
|
||||
|
||||
namespace QL;
|
||||
use phpQuery;
|
||||
use QL\Dom\Query;
|
||||
use Tightenco\Collect\Support\Collection;
|
||||
use Closure;
|
||||
use QL\Services\MultiRequestService;
|
||||
|
||||
|
||||
/**
|
||||
* Class QueryList
|
||||
* @package QL
|
||||
*
|
||||
* @method string getHtml($rel = true)
|
||||
* @method QueryList setHtml($html)
|
||||
* @method QueryList html($html)
|
||||
* @method Dom\Elements find($selector)
|
||||
* @method QueryList rules(array $rules)
|
||||
* @method QueryList range($range)
|
||||
* @method QueryList removeHead()
|
||||
* @method QueryList query(Closure $callback = null)
|
||||
* @method Collection getData(Closure $callback = null)
|
||||
* @method Array queryData(Closure $callback = null)
|
||||
* @method QueryList setData(Collection $data)
|
||||
* @method QueryList encoding(string $outputEncoding,string $inputEncoding = null)
|
||||
* @method QueryList get($url,$args = null,$otherArgs = [])
|
||||
* @method QueryList post($url,$args = null,$otherArgs = [])
|
||||
* @method QueryList postJson($url,$args = null,$otherArgs = [])
|
||||
* @method MultiRequestService multiGet($urls)
|
||||
* @method MultiRequestService multiPost($urls)
|
||||
* @method QueryList use($plugins,...$opt)
|
||||
* @method QueryList pipe(Closure $callback = null)
|
||||
*/
|
||||
class QueryList
|
||||
{
|
||||
protected $query;
|
||||
protected $kernel;
|
||||
protected static $instance = null;
|
||||
|
||||
/**
|
||||
* QueryList constructor.
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this->query = new Query($this);
|
||||
$this->kernel = (new Kernel($this))->bootstrap();
|
||||
Config::getInstance()->bootstrap($this);
|
||||
}
|
||||
|
||||
public function __call($name, $arguments)
|
||||
{
|
||||
if(method_exists($this->query,$name)){
|
||||
$result = $this->query->$name(...$arguments);
|
||||
}else{
|
||||
$result = $this->kernel->getService($name)->call($this,...$arguments);
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
|
||||
public static function __callStatic($name, $arguments)
|
||||
{
|
||||
$instance = new self();
|
||||
return $instance->$name(...$arguments);
|
||||
}
|
||||
|
||||
public function __destruct()
|
||||
{
|
||||
$this->destruct();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the QueryList single instance
|
||||
*
|
||||
* @return QueryList
|
||||
*/
|
||||
public static function getInstance()
|
||||
{
|
||||
self::$instance || self::$instance = new self();
|
||||
return self::$instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the Config instance
|
||||
* @return null|Config
|
||||
*/
|
||||
public static function config()
|
||||
{
|
||||
return Config::getInstance();
|
||||
}
|
||||
|
||||
/**
|
||||
* Destruction of resources
|
||||
*/
|
||||
public function destruct()
|
||||
{
|
||||
unset($this->query);
|
||||
unset($this->kernel);
|
||||
}
|
||||
|
||||
/**
|
||||
* Destroy all documents
|
||||
*/
|
||||
public static function destructDocuments()
|
||||
{
|
||||
phpQuery::$documents = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Bind a custom method to the QueryList object
|
||||
*
|
||||
* @param string $name Invoking the name
|
||||
* @param Closure $provide Called method
|
||||
* @return $this
|
||||
*/
|
||||
public function bind(string $name,Closure $provide)
|
||||
{
|
||||
$this->kernel->bind($name,$provide);
|
||||
return $this;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
<?php
|
||||
/**
|
||||
* Created by PhpStorm.
|
||||
* User: Jaeger <JaegerCode@gmail.com>
|
||||
* Date: 2017/9/20
|
||||
* 编码转换服务
|
||||
*/
|
||||
|
||||
namespace QL\Services;
|
||||
|
||||
use QL\QueryList;
|
||||
|
||||
class EncodeService
|
||||
{
|
||||
public static function convert(QueryList $ql,string $outputEncoding,string $inputEncoding = null)
|
||||
{
|
||||
$html = $ql->getHtml();
|
||||
$inputEncoding || $inputEncoding = self::detect($html);
|
||||
$html = iconv($inputEncoding,$outputEncoding.'//IGNORE',$html);
|
||||
$ql->setHtml($html);
|
||||
return $ql;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to detect the encoding
|
||||
* @param $string
|
||||
* @return bool|false|mixed|string
|
||||
*/
|
||||
public static function detect($string)
|
||||
{
|
||||
$charset=mb_detect_encoding($string, array('ASCII', 'GB2312', 'GBK', 'UTF-8'),true);
|
||||
if(strtolower($charset)=='cp936')
|
||||
$charset='GBK';
|
||||
return $charset;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
<?php
|
||||
/**
|
||||
* Created by PhpStorm.
|
||||
* User: Jaeger <JaegerCode@gmail.com>
|
||||
* Date: 2017/9/22
|
||||
*/
|
||||
|
||||
namespace QL\Services;
|
||||
|
||||
use GuzzleHttp\Cookie\CookieJar;
|
||||
use Jaeger\GHttp;
|
||||
use QL\QueryList;
|
||||
|
||||
class HttpService
|
||||
{
|
||||
protected static $cookieJar = null;
|
||||
|
||||
public static function getCookieJar()
|
||||
{
|
||||
if(self::$cookieJar == null)
|
||||
{
|
||||
self::$cookieJar = new CookieJar();
|
||||
}
|
||||
return self::$cookieJar;
|
||||
}
|
||||
|
||||
public static function get(QueryList $ql,$url,$args = null,$otherArgs = [])
|
||||
{
|
||||
$otherArgs = array_merge([
|
||||
'cookies' => self::getCookieJar(),
|
||||
'verify' => false
|
||||
],$otherArgs);
|
||||
$html = GHttp::get($url,$args,$otherArgs);
|
||||
$ql->setHtml($html);
|
||||
return $ql;
|
||||
}
|
||||
|
||||
public static function post(QueryList $ql,$url,$args = null,$otherArgs = [])
|
||||
{
|
||||
$otherArgs = array_merge([
|
||||
'cookies' => self::getCookieJar(),
|
||||
'verify' => false
|
||||
],$otherArgs);
|
||||
$html = GHttp::post($url,$args,$otherArgs);
|
||||
$ql->setHtml($html);
|
||||
return $ql;
|
||||
}
|
||||
|
||||
public static function postJson(QueryList $ql,$url,$args = null,$otherArgs = [])
|
||||
{
|
||||
$otherArgs = array_merge([
|
||||
'cookies' => self::getCookieJar(),
|
||||
'verify' => false
|
||||
],$otherArgs);
|
||||
$html = GHttp::postJson($url,$args,$otherArgs);
|
||||
$ql->setHtml($html);
|
||||
return $ql;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
<?php
|
||||
/**
|
||||
* Created by PhpStorm.
|
||||
* User: Jaeger <JaegerCode@gmail.com>
|
||||
* Date: 18/12/10
|
||||
* Time: 下午7:05
|
||||
*/
|
||||
|
||||
namespace QL\Services;
|
||||
|
||||
|
||||
use Jaeger\GHttp;
|
||||
use Closure;
|
||||
use GuzzleHttp\Psr7\Response;
|
||||
use QL\QueryList;
|
||||
use GuzzleHttp\Exception\RequestException;
|
||||
|
||||
/**
|
||||
* Class MultiRequestService
|
||||
* @package QL\Services
|
||||
*
|
||||
* @method MultiRequestService withHeaders($headers)
|
||||
* @method MultiRequestService withOptions($options)
|
||||
* @method MultiRequestService concurrency($concurrency)
|
||||
*/
|
||||
class MultiRequestService
|
||||
{
|
||||
protected $ql;
|
||||
protected $multiRequest;
|
||||
protected $method;
|
||||
|
||||
public function __construct(QueryList $ql,$method,$urls)
|
||||
{
|
||||
$this->ql = $ql;
|
||||
$this->method = $method;
|
||||
$this->multiRequest = GHttp::multiRequest($urls);
|
||||
}
|
||||
|
||||
public function __call($name, $arguments)
|
||||
{
|
||||
$this->multiRequest = $this->multiRequest->$name(...$arguments);
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function success(Closure $success)
|
||||
{
|
||||
$this->multiRequest = $this->multiRequest->success(function(Response $response, $index) use($success){
|
||||
$this->ql->setHtml((String)$response->getBody());
|
||||
$success($this->ql,$response, $index);
|
||||
});
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function error(Closure $error)
|
||||
{
|
||||
$this->multiRequest = $this->multiRequest->error(function(RequestException $reason, $index) use($error){
|
||||
$error($this->ql,$reason, $index);
|
||||
});
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function send()
|
||||
{
|
||||
$this->multiRequest->{$this->method}();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
<?php
|
||||
/**
|
||||
* Created by PhpStorm.
|
||||
* User: Jaeger <JaegerCode@gmail.com>
|
||||
* Date: 2017/9/22
|
||||
*/
|
||||
|
||||
namespace QL\Services;
|
||||
|
||||
use QL\QueryList;
|
||||
|
||||
class PluginService
|
||||
{
|
||||
public static function install(QueryList $queryList, $plugins, ...$opt)
|
||||
{
|
||||
if(is_array($plugins))
|
||||
{
|
||||
foreach ($plugins as $plugin) {
|
||||
$plugin::install($queryList);
|
||||
}
|
||||
}else{
|
||||
$plugins::install($queryList,...$opt);
|
||||
}
|
||||
return $queryList;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user