diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..bd95349 --- /dev/null +++ b/.env.example @@ -0,0 +1 @@ +ELASTIC_HOST=127.0.0.1:9200 diff --git a/CHANGELOG-1.0.md b/CHANGELOG-1.0.md new file mode 100755 index 0000000..dbde8b7 --- /dev/null +++ b/CHANGELOG-1.0.md @@ -0,0 +1,6 @@ +# Release Notes for 1.0.x + +## [Unreleased] + +- Upgrading the library to Elastic Search V6 + diff --git a/README.md b/README.md index 3f0c122..433c687 100755 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Baka Phalcon Elastic Search +# Baka Phalcon ElasticSearch Phalcon Elastic Search package to index / query model with relationship easily @@ -9,9 +9,13 @@ Phalcon Elastic Search package to index / query model with relationship easily 2. [Search](#markdown-header-QueryParser) 4. [Testing](#markdown-header-QueryParser-Extended) +## ElasticSearch 6.x configuration +- `indices.query.bool.max_clause_count' => 1000000` if you are doing long searches for lots of conditionals +- [Elasticsearch 6.x Cheatsheet](http://elasticsearch-cheatsheet.jolicode.com/) + ## Installing Packages: -- `"elasticsearch/elasticsearch": "~2.0@beta"` +- `"elasticsearch/elasticsearch": "^6.1"` - `"baka/database": "dev-master"` - `"phalcon/incubator": "~3.0","` @@ -137,4 +141,4 @@ Example ``` codecept run -``` \ No newline at end of file +``` diff --git a/composer.json b/composer.json index 7240f65..2a568cf 100755 --- a/composer.json +++ b/composer.json @@ -10,17 +10,18 @@ ], "minimum-stability": "dev", "require": { - "php": ">=7.1", + "php": ">=7.2", "ext-phalcon": ">=3.0.0", "vlucas/phpdotenv": "^2.0", "phalcon/incubator": "~3.3", "monolog/monolog": "^1.16", - "baka/database": "~0.1.0", - "baka/http": "^0.1.1", - "elasticsearch/elasticsearch": "^2.3.1" + "baka/database": "^0.5", + "baka/http": "^0.5", + "elasticsearch/elasticsearch": "^6.1" }, "require-dev": { "codeception/verify": "*", + "codeception/codeception": "2.4", "duncan3dc/fork-helper": "^2.2.0" }, "autoload": { diff --git a/phpcs.xml b/phpcs.xml new file mode 100755 index 0000000..779ac23 --- /dev/null +++ b/phpcs.xml @@ -0,0 +1,57 @@ + + + Phalcon Coding Standards + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + tests/cli + tests/integration + tests/unit + tests/_data/fixtures/Traits + tests/_support/Helper + \ No newline at end of file diff --git a/src/Client.php b/src/Client.php new file mode 100644 index 0000000..af567c9 --- /dev/null +++ b/src/Client.php @@ -0,0 +1,69 @@ +host = $host; + } + + /** + * Given a SQL search the elastic indices. + * + * @param string $sql + * @return void + */ + public function findBySql(string $sql): array + { + $client = new GuzzleClient([ + 'base_uri' => $this->host, + ]); + + // since 6.x we need to use POST + $response = $client->post('/_sql', [ + 'body' => trim($sql), + 'headers' => [ + 'content-type' => 'application/json', + 'Accept' => 'application/json' + ], + ]); + + //get the response in a array + $results = json_decode($response->getBody()->getContents(), true); + + if ($results['hits']['total'] == 0) { + return []; + } + + return $this->getResults($results); + } + + /** + * Given the elastic results, return only the data. + * + * @param array $resonse + * @return array + */ + private function getResults(array $results): array + { + $data = []; + foreach ($results['hits']['hits'] as $result) { + $data[] = $result['_source']; + } + + return $data; + } +} diff --git a/src/Contracts/CustomFiltersSchemaTrait.php b/src/Contracts/CustomFiltersSchemaTrait.php new file mode 100644 index 0000000..30e87f1 --- /dev/null +++ b/src/Contracts/CustomFiltersSchemaTrait.php @@ -0,0 +1,67 @@ +elastic->indices()->getMapping([ + 'index' => $index, + ]); + + $mapping = array_shift($mapping); + + //if we dont find mapping return empty + if (!isset($mapping['mappings'])) { + return []; + } + + $mapping = array_shift($mapping); + + //we only need the infro fromt he properto onward + //we want the result to be in a linear array so we pass it by reference + $result = []; + $results = $this->mappingToArray(array_shift($mapping)['properties'], null, $result); + rsort($results); //rever order? + return $results; + } + + /** + * Generate the array map fromt he elastic search mapping. + * + * @param array $mappings + * @param string $parent + * @param array $result + * @return array + */ + protected function mappingToArray(array $mappings, string $parent = null, array &$result): array + { + foreach ($mappings as $key => $mapping) { + if (isset($mapping['type']) && $mapping['type'] != 'nested') { + $result[] = $parent . $key; + } elseif (isset($mapping['type']) && $mapping['type'] == 'nested' && is_array($mapping)) { + //setup key + $parent .= $key . '.'; + + //look for more records + $this->mappingToArray($mapping['properties'], $parent, $result); + + //so we finisht with a child , we need to change the parent to one back + $parentExploded = explode('.', $parent); + $parent = count($parentExploded) > 2 ? $parentExploded[0] . '.' : null; + } + } + + return $result; + } +} diff --git a/src/IndexBuilderTask.php b/src/Contracts/IndexBuilderTaskTrait.php old mode 100755 new mode 100644 similarity index 94% rename from src/IndexBuilderTask.php rename to src/Contracts/IndexBuilderTaskTrait.php index 254b562..7f6f020 --- a/src/IndexBuilderTask.php +++ b/src/Contracts/IndexBuilderTaskTrait.php @@ -1,151 +1,151 @@ -createIndices($model, $maxDepth, $nestedLimit); - } - } - - /** - * Action Descriptor - * - * Command: insert - * Description: Create the elasticsearch index all info for a model. - * - * php cli/app.php elasticsearch insert modelName 0 1 - * - * @param string $model - * @param int $maxDepth - * - * @return void - */ - public function insertAction($params): void - { - list($model, $maxDepth) = $params + ['', 3]; - - if (!empty($model)) { - // Get model - $model = $this->config->namespace->models . '\\' . $model; - // Get model's records - $records = $model::find('is_deleted = 0'); - // Get elasticsearch class handler instance - $elasticsearch = new IndexBuilder(); - - foreach ($records as $record) { - $elasticsearch->indexDocument($record, $maxDepth); - } - } - } - - /** - * Elastic Search insert Queue - * - * php cli/app.php elasticsearch queue queueName - * - * @return void - */ - public function queueAction(array $queueName): void - { - try { - // Check that a queue name has been provided - if (empty($queueName)) { - throw new Throwable('You have to define a queue name.'); - } - - // Start the queue - $queue = new BeanstalkExtended([ - 'host' => $this->config->beanstalk->host, - 'prefix' => $this->config->beanstalk->prefix, - ]); - - // Variables needed by the annonymous function - $config = $this->config; - $di = \Phalcon\DI\FactoryDefault::getDefault(); - - //call queue tube - $queue->addWorker($queueName[0], function (Job $job) use ($di, $config) { - try { - // Get the job - $record = $job->getBody(); - - $model = $record['model']; - $id = $record['id']; - $maxDepth = isset($record['maxDepth']) ? $record['maxDepth'] : 1; - - // Get model - $model = $this->config->namespace->models . '\\' . $model; - - if (!class_exists($model)) { - $this->log->error('Queue Elastic class doesnt exit ' . $model); - return; - } - - $this->log->info('Processing ' . $model . ' Id: ' . $id); - // Get model's records - if ($record = $model::findFirst($id)) { - // Get elasticsearch class handler instance - $elasticsearch = new IndexBuilder(); - - //insert into elastic - $elasticsearch->indexDocument($record, $maxDepth); - } - } catch (Throwable $e) { - echo $e->getTraceAsString(); - $this->log->error($e->getTraceAsString()); - } - - // It's very important to send the right exit code! - exit(0); - }); - - // Start processing queues - $queue->doWork(); - } catch (Throwable $e) { - $this->log->error($e->getMessage()); - } - } -} +createIndices($model, $maxDepth, $nestedLimit); + } + } + + /** + * Action Descriptor + * + * Command: insert + * Description: Create the elasticsearch index all info for a model. + * + * php cli/app.php elasticsearch insert modelName 0 1 + * + * @param string $model + * @param int $maxDepth + * + * @return void + */ + public function insertAction($params): void + { + list($model, $maxDepth) = $params + ['', 3]; + + if (!empty($model)) { + // Get model + $model = $this->config->namespace->models . '\\' . $model; + // Get model's records + $records = $model::find('is_deleted = 0'); + // Get elasticsearch class handler instance + $elasticsearch = new IndexBuilder(); + + foreach ($records as $record) { + $elasticsearch->indexDocument($record, $maxDepth); + } + } + } + + /** + * Elastic Search insert Queue + * + * php cli/app.php elasticsearch queue queueName + * + * @return void + */ + public function queueAction(array $queueName): void + { + try { + // Check that a queue name has been provided + if (empty($queueName)) { + throw new Throwable('You have to define a queue name.'); + } + + // Start the queue + $queue = new BeanstalkExtended([ + 'host' => $this->config->beanstalk->host, + 'prefix' => $this->config->beanstalk->prefix, + ]); + + // Variables needed by the annonymous function + $config = $this->config; + $di = \Phalcon\DI\FactoryDefault::getDefault(); + + //call queue tube + $queue->addWorker($queueName[0], function (Job $job) use ($di, $config) { + try { + // Get the job + $record = $job->getBody(); + + $model = $record['model']; + $id = $record['id']; + $maxDepth = isset($record['maxDepth']) ? $record['maxDepth'] : 1; + + // Get model + $model = $this->config->namespace->models . '\\' . $model; + + if (!class_exists($model)) { + $this->log->error('Queue Elastic class doesnt exit ' . $model); + return; + } + + $this->log->info('Processing ' . $model . ' Id: ' . $id); + // Get model's records + if ($record = $model::findFirst($id)) { + // Get elasticsearch class handler instance + $elasticsearch = new IndexBuilder(); + + //insert into elastic + $elasticsearch->indexDocument($record, $maxDepth); + } + } catch (Throwable $e) { + echo $e->getTraceAsString(); + $this->log->error($e->getTraceAsString()); + } + + // It's very important to send the right exit code! + exit(0); + }); + + // Start processing queues + $queue->doWork(); + } catch (Throwable $e) { + $this->log->error($e->getMessage()); + } + } +} diff --git a/src/IndexBuilder.php b/src/IndexBuilder.php index c133746..fcae2bd 100755 --- a/src/IndexBuilder.php +++ b/src/IndexBuilder.php @@ -2,12 +2,12 @@ namespace Baka\Elasticsearch; -use \Baka\Database\ModelCustomFields; -use \Baka\Database\CustomFields\CustomFields; -use \Elasticsearch\ClientBuilder as Client; -use \Exception; -use \Phalcon\Db\Column; -use \Phalcon\Mvc\Model; +use Baka\Elasticsearch\Model as ModelCustomFields; +use Baka\Database\CustomFields\CustomFields; +use Elasticsearch\ClientBuilder as Client; +use Exception; +use Phalcon\Db\Column; +use Phalcon\Mvc\Model; class IndexBuilder { @@ -86,6 +86,44 @@ protected static function checks(string $model): string return $model; } + /** + * Get the general settings for our predefind indices + * + * @param integer $nestedLimit + * @return array + */ + protected static function getIndicesSettings(int $nestedLimit): array + { + return [ + 'index.mapping.nested_fields.limit' => $nestedLimit, + 'max_result_window' => 50000, + 'analysis' => [ + 'analyzer' => [ + 'lowercase' => [ + 'type' => 'custom', + 'tokenizer' => 'keyword', + 'filter' => ['lowercase'], + ], + ], + ] + ]; + } + + /** + * Check if the index exist + * + * @param string $model + * @return void + */ + public static function existIndices(string $model): bool + { + // Run checks to make sure everything is in order. + $modelPath = self::checks($model); + $model = strtolower(str_replace(['_', '-'], '', $model)); + + return self::$client->indices()->exists(['index' => $model]); + } + /** * Create an index for a model * @@ -109,20 +147,7 @@ public static function createIndices(string $model, int $maxDepth = 3, int $nest $params = [ 'index' => $model, 'body' => [ - 'settings' => [ - 'index.mapping.nested_fields.limit' => $nestedLimit, - 'max_result_window' => 50000, - 'index.query.bool.max_clause_count' => 1000000, - 'analysis' => [ - 'analyzer' => [ - 'lowercase' => [ - 'type' => 'custom', - 'tokenizer' => 'keyword', - 'filter' => ['lowercase'], - ], - ], - ], - ], + 'settings' => self::getIndicesSettings($nestedLimit), 'mappings' => [ $model => [ 'properties' => [], @@ -157,8 +182,10 @@ public static function createIndices(string $model, int $maxDepth = 3, int $nest // Call to get the information from related models. self::getRelatedParams($params['body']['mappings'][$model]['properties'], $modelPath, $modelPath, 1, $maxDepth); - // Delete the index before creating it again - // @TODO move this to its own function + /** + * Delete the index before creating it again + * @todo move this to its own function + */ if (self::$client->indices()->exists(['index' => $model])) { self::$client->indices()->delete(['index' => $model]); } @@ -230,7 +257,7 @@ public static function deleteDocument(Model $object): array protected static function getFieldsTypes(string $modelPath): array { // Get the columns description. - $columns = self::$di->getDb()->describeColumns($modelPath); + $columns = self::$di->getDb()->describeColumns(strtolower($modelPath)); // Define a fields array $fields = []; @@ -246,7 +273,7 @@ protected static function getFieldsTypes(string $modelPath): array case Column::TYPE_TEXT: case Column::TYPE_VARCHAR: case Column::TYPE_CHAR: - $fields[$column->getName()] = 'string'; + $fields[$column->getName()] = 'text'; break; case Column::TYPE_DATE: // We define a format for date fields. @@ -351,10 +378,10 @@ protected static function getCustomParams(array &$params, string $modelPath) : v foreach ($customFields as $field) { $type = [ - 'type' => 'string', + 'type' => 'text', 'analyzer' => 'lowercase', ]; - if ($field->type == 'date') { + if ($field['type'] == 'date') { $type = [ 'type' => 'date', 'format' => 'yyyy-MM-dd', @@ -362,7 +389,7 @@ protected static function getCustomParams(array &$params, string $modelPath) : v ]; } - $params['custom_fields']['properties'][$field->name] = $type; + $params['custom_fields']['properties'][$field['name']] = $type; } } } diff --git a/src/IndexBuilderStructure.php b/src/IndexBuilderStructure.php index 8b6a1e4..b41439a 100755 --- a/src/IndexBuilderStructure.php +++ b/src/IndexBuilderStructure.php @@ -3,7 +3,8 @@ namespace Baka\Elasticsearch; use Exception; -use \Phalcon\Mvc\Model; +use Phalcon\Mvc\Model; +use ReflectionClass; class IndexBuilderStructure extends IndexBuilder { @@ -21,24 +22,8 @@ protected static function checks(string $model) : string // Call the initializer. self::initialize(); - // Check that there is a configuration for namespaces. - if (!$config = self::$di->getConfig()->get('namespace')) { - throw new Exception('Please add your namespace definitions to the configuration.'); - } - - // Check that there is a namespace definition for modules. - if (!array_key_exists('models', $config)) { - throw new Exception('Please add the namespace definition for your models.'); - } - - // Get the namespace. - $namespace = $config['elasticIndex']; - - // We have to do some work with the model name before we continue to avoid issues. - $model = str_replace(' ', '', ucwords(str_replace(['_', '-'], ' ', $model))); - // Check that the defined model actually exists. - if (!class_exists($model = $namespace . '\\' . $model)) { + if (!class_exists($model)) { throw new Exception('The specified model does not exist.'); } @@ -62,7 +47,7 @@ public static function indexDocument(Model $object, int $maxDepth = 3) : array $modelReflection = (new \ReflectionClass($object)); $document = $object->document(); - $indexName = is_null(static::$indexName) ? strtolower($modelReflection->getShortName()) : static::$indexName; + $indexName = static::$indexName ?: strtolower($modelReflection->getShortName()); $params = [ 'index' => $indexName, @@ -89,7 +74,7 @@ public static function deleteDocument(Model $object) : array $modelReflection = (new \ReflectionClass($object)); $object->document(); - $indexName = is_null(static::$indexName) ? strtolower($modelReflection->getShortName()) : static::$indexName; + $indexName = static::$indexName ?: strtolower($modelReflection->getShortName()); $params = [ 'index' => $indexName, @@ -111,6 +96,24 @@ public static function setIndexName(string $indexName): void static::$indexName = strtolower($indexName); } + /** + * Check if the index exist + * + * @param string $model + * @return void + */ + public static function existIndices(string $model): bool + { + // Run checks to make sure everything is in order. + $modelPath = self::checks($model); + + // We need to instance the model in order to access some of its properties. + $modelInstance = new $modelPath(); + $model = self::$indexName ?: strtolower(str_replace(['_', '-'], '', (new ReflectionClass($modelInstance))->getShortName())); + + return self::$client->indices()->exists(['index' => $model]); + } + /** * Create an index for a model * @@ -131,26 +134,13 @@ public static function createIndices(string $model, int $maxDepth = 3, int $nest $columns = $modelInstance->structure(); // Set the model variable for use as a key. - $model = is_null(self::$indexName) ? strtolower(str_replace(['_', '-'], '', $model)) : self::$indexName; + $model = self::$indexName ?: strtolower(str_replace(['_', '-'], '', (new ReflectionClass($modelInstance))->getShortName())); // Define the initial parameters that will be sent to Elasticsearch. $params = [ 'index' => $model, 'body' => [ - 'settings' => [ - 'index.mapping.nested_fields.limit' => $nestedLimit, - 'max_result_window' => 50000, - 'index.query.bool.max_clause_count' => 1000000, - 'analysis' => [ - 'analyzer' => [ - 'lowercase' => [ - 'type' => 'custom', - 'tokenizer' => 'keyword', - 'filter' => ['lowercase'], - ], - ], - ], - ], + 'settings' => self::getIndicesSettings($nestedLimit), 'mappings' => [ $model => [ 'properties' => [], @@ -184,7 +174,6 @@ public static function createIndices(string $model, int $maxDepth = 3, int $nest if (self::$client->indices()->exists(['index' => $model])) { self::$client->indices()->delete(['index' => $model]); } - return self::$client->indices()->create($params); } diff --git a/src/Indices.php b/src/Indices.php index dc3cbf8..99819f7 100755 --- a/src/Indices.php +++ b/src/Indices.php @@ -6,7 +6,7 @@ abstract class Indices extends \Baka\Database\Model { - protected $text = 'string'; + protected $text = 'text'; protected $integer = 'integer'; protected $bigInt = 'long'; protected $dateNormal = ['date', 'yyyy-MM-dd']; @@ -14,7 +14,7 @@ abstract class Indices extends \Baka\Database\Model protected $decimal = 'float'; /** - * Set the Id + * Set the Id. * * @param integer $id * @return void @@ -25,21 +25,21 @@ public function setId(int $id): void } /** - * Define de structure for this index in elastic search + * Define de structure for this index in elastic search. * * @return array */ abstract public function structure() : array; /** - * Set the data of the current index + * Set the data of the current index. * * @return stdClass */ abstract public function data() : stdClass; /** - * Given the object of the class we return a array document + * Given the object of the class we return a array document. * * @return array */ diff --git a/src/Model.php b/src/Model.php index 598b091..1f08b05 100755 --- a/src/Model.php +++ b/src/Model.php @@ -2,8 +2,12 @@ namespace Baka\Elasticsearch; +use Baka\Database\Contracts\CustomFields\CustomFieldsTrait; + class Model extends \Baka\Database\Model { + use CustomFieldsTrait; + protected $elasticMaxDepth = 3; /** diff --git a/src/SearchTrait.php b/src/SearchTrait.php deleted file mode 100755 index 32df601..0000000 --- a/src/SearchTrait.php +++ /dev/null @@ -1,282 +0,0 @@ -request->getQuery('q'); - $nestedQuery = $this->request->getQuery('nq'); - $fields = $this->request->getQuery('fields', null, '*'); - $page = $this->request->getQuery('page', null, 1); - - $limit = 3000; - - if ($this->request->hasQuery('limit')) { - $limit = $this->request->getQuery('limit', null, 50); - } elseif ($this->request->hasQuery('per_page')) { - $limit = $this->request->getQuery('per_page', null, 50); - } - if ($paramLimit) { - $limit = $paramLimit; - } - - $sort = $this->request->getQuery('sort', null, 'id ASC'); - if (empty($sort)) { - $sort = 'id ASC'; - } - - $sort = str_replace('|', ' ', $sort); - $offset = ($page - 1) * $limit; - - $sql = ''; - - $operators = [ - ':' => 'LIKE', - '>' => '>=', - '<' => '<=', - ]; - - if (!empty($query)) { - $query = $this->parseSearchParameters($query); - - foreach ($query as $field => $value) { - @list($field, $operator, $value) = $value; - $operator = isset($operators[$operator]) ? $operators[$operator] : null; - - if (trim($value) != '') { - if ($value == '%%') { - $sql .= ' AND (' . $field . ' IS NULL'; - $sql .= ' OR ' . $field . ' = "")'; - } else { - if (strpos($value, '|')) { - $value = explode('|', $value); - } else { - $value = [$value]; - } - } - - foreach ($value as $k => $v) { - $v = strtolower(str_replace('%', '*', $v)); - - if (!$k) { - $sql .= " AND ({$field} {$operator} '{$v}'"; - } else { - $sql .= " OR {$field} {$operator} '{$v}'"; - } - } - - $sql .= ')'; - } - } - } - - if (!empty($nestedQuery)) { - $nestedQuery = $this->parseSearchParameters($nestedQuery); - - foreach ($nestedQuery as $field => $value) { - @list($field, $operator, $value) = $value; - $operator = isset($operators[$operator]) ? $operators[$operator] : null; - - if (trim($value) !== '') { - if ($value == '%%') { - $sql .= ' AND (nested(' . $field . ') IS NULL'; - $sql .= ' OR nested(' . $field . ') = "")'; - } else { - if (strpos($value, '|')) { - $value = explode('|', $value); - } else { - $value = [$value]; - } - } - - foreach ($value as $k => $v) { - $v = strtolower(str_replace('%', '*', $v)); - - if (!$k) { - $sql .= " AND (nested({$field}) {$operator} '{$v}'"; - } else { - $sql .= " OR nested({$field}) {$operator} '{$v}'"; - } - } - - $sql .= ')'; - } - } - } - - empty($sql) ?: $sql = ' WHERE' . substr($sql, 4); - - $resultsSql = 'SELECT ' . $fields . ' FROM ' . $model . $sql . ' ORDER BY ' . $sort . ($withLimit ? ' LIMIT ' . $offset . ', ' . $limit : ''); - - $client = new \GuzzleHttp\Client(); - $response = $client->get('http://' . $this->config->elasticSearch['hosts'][0] . '/_sql?sql=' . urlencode($resultsSql)); - $results = json_decode($response->getBody()->getContents(), true); - - if ($results['hits']['total'] == 0) { - throw new \Exception('No records were found.'); - } - - $data = []; - foreach ($results['hits']['hits'] as $result) { - $result['_source']['is_checked'] = 1; - if ($filterId != 0) { - foreach ($checkedIncludes as $key => $value) { - if ($result['_source']['id'] == $key) { - $result['_source']['is_checked'] = $value; - } - } - } - $data[] = $result['_source']; - } - - /** - * Sort the response with PHP since elastic sql has problesm - * @todo check this shit - */ - $dataSort = []; - $sortFields = explode(',', $sort); - foreach ($sortFields as $key => $sortField) { - $field = explode(' ', $sortField); - $dataSort[] = $field[0]; - $dataSort[] = isset($field[1]) && strtolower($field[1]) == 'asc' ? SORT_ASC : SORT_DESC; - } - - //get the data sorted - $data = $this->orderBy($data, $dataSort); - - $pages = $this->paginate( - $data, - $results['hits']['total'], - $page, - $offset, - $limit - ); - - return $this->response($pages); - } - - /** - * Order the search by it id - * we do order in php since its faster then using mysql - * - * @param array $data - * @param array $args - * @return array - */ - protected function orderBy($data, $args): array - { - /** - * naveget the response and sort it by the field - */ - foreach ($args as $n => $field) { - if (is_string($field)) { - $tmp = []; - foreach ($data as $key => $row) { - $fieldPath = explode('.', $field); - $fieldVal = $row; - foreach ($fieldPath as $value) { - if (is_array($fieldVal) && array_key_exists($value, $fieldVal)) { - $fieldVal = $fieldVal[$value]; - } else { - $fieldVal = ''; - } - } - $tmp[$key] = $fieldVal; - } - $args[$n] = $tmp; - } - } - - $args[] = &$data; - call_user_func_array('array_multisort', $args); - - return array_pop($args); - } - - /** - * Paginate array - * - * @param array $items The items - * @param integer $totalItems The total items - * @param integer $page The page - * @param integer $offset The offset - * @param integer $limit The limit - * - * @return \stdClass - */ - protected function paginate($items, $totalItems, $page, $offset, $limit): stdClass - { - $limit = (int) $limit; - $offset = (int) $offset; - $totalPages = ceil($totalItems / $limit); - - $next = $page < $totalPages ? $page + 1 : ''; - $prev = $page > 1 ? $page - 1 : ''; - - $pagination = new stdClass(); - $pagination->total = (int) $totalItems; - $pagination->per_page = (int) $limit; - $pagination->current_page = (int) $page; - $pagination->last_page = $totalPages; - $pagination->next_page_url = ''; - $pagination->prev_page_url = ''; - $pagination->from = $offset + 1; - $pagination->to = $offset + $limit; - - if ($pagination->to > $totalItems) { - $pagination->to = $totalItems; - } - - $pagination->data = $items; - - $pagination->total_pages = $totalPages; - - return $pagination; - } - - /** - * Parse the search query, overwrite the parent furnction - * - * @param string $unparsed - * @return array - */ - protected function parseSearchParameters($unparsed): array - { - // $unparsed = urldecode($unparsed); - // Strip parens that come with the request string - $unparsed = trim($unparsed, '()'); - - // Now we have an array of "key:value" strings. - $splitFields = explode(',', $unparsed); - $mapped = []; - - // Split the strings at their colon, set left to key, and right to value. - foreach ($splitFields as $field) { - $splitField = preg_split('#(:|>|<)#', $field, -1, PREG_SPLIT_DELIM_CAPTURE); - - // TEMP: Fix for strings that contain semicolon - if (count($splitField) > 3) { - $splitField[2] = implode('', array_splice($splitField, 2)); - } - - $mapped[] = $splitField; - } - - return $mapped; - } -} diff --git a/tests/_bootstrap.php b/tests/_bootstrap.php index 996901c..4190a04 100755 --- a/tests/_bootstrap.php +++ b/tests/_bootstrap.php @@ -7,5 +7,24 @@ require __DIR__ . '/../vendor/autoload.php'; require __DIR__ . '/unit/PhalconUnitTestCase.php'; + +if (!defined('ROOT_DIR')) { + define('ROOT_DIR', dirname(__DIR__) . '/'); +} + +//load classes +$loader = new \Phalcon\Loader(); +$loader->registerNamespaces( + [ + 'Baka\Database' => ROOT_DIR . '..\database\src', + 'Baka\Elasticsearch' => ROOT_DIR . 'src', + 'Test\Model' => ROOT_DIR . 'tests\_support\Model', + 'Test\Indices' => ROOT_DIR . 'tests\_support\Indices', + 'Baka\Auth' => ROOT_DIR . '..\auth\src\\', + ] +); + +$loader->register(); + $dotenv = new Dotenv\Dotenv(__DIR__ . '/../'); $dotenv->load(); diff --git a/tests/_support/Model/Leads.php b/tests/_support/Model/Leads.php new file mode 100644 index 0000000..20fa87a --- /dev/null +++ b/tests/_support/Model/Leads.php @@ -0,0 +1,21 @@ +leads_id = $id; + } +} diff --git a/tests/unit/CustomFilterSchemaTest.php b/tests/unit/CustomFilterSchemaTest.php new file mode 100644 index 0000000..9b3fe80 --- /dev/null +++ b/tests/unit/CustomFilterSchemaTest.php @@ -0,0 +1,32 @@ +elastic = $this->getDI()->getElastic(); + + $mapping = $this->getSchema('leads'); + + $this->assertTrue(!empty($mapping)); + $this->assertTrue(array_search('id', $mapping) > 0); + } +} diff --git a/tests/unit/IndicesModelTest.php b/tests/unit/IndicesModelTest.php new file mode 100644 index 0000000..1457635 --- /dev/null +++ b/tests/unit/IndicesModelTest.php @@ -0,0 +1,115 @@ +elastic = $this->getDI()->getElastic(); + + //create index + $this->createIndexAction([ + 'Leads', //model + '1' //depth + ]); + + $mapping = $this->getSchema('leads'); + + $this->assertTrue(array_search('id', $mapping) > 0); + } + + /** + * Test inserting data to elastic search froma module. + * + * @return void + */ + public function testInsertAllDataFromModel() + { + //cli need the config + $this->config = $this->getDI()->getConfig(); + $this->elastic = $this->getDI()->getElastic(); + + $this->insertAction([ + 'Leads', //model + 1, //depth + ]); + + $lead = Leads::findFirst(); + $params = [ + 'index' => 'leads', + 'type' => 'leads', + 'id' => $lead->getId() + ]; + + $response = $this->elastic->get($params); + + $this->assertTrue($response['_source']['id'] == $lead->getId()); + } + + /** + * Insert just 1 record. + * + * @return void + */ + public function testInsertOneDocumentFromARecordModel() + { + //cli need the config + $this->config = $this->getDI()->getConfig(); + $this->elastic = $this->getDI()->getElastic(); + + $lead = Leads::findFirst(); + + // Get elasticsearch class handler instance + $elasticsearch = new IndexBuilder(); + + //insert into elastic + $elasticsearch->indexDocument($lead, 1); //depth + + $params = [ + 'index' => 'leads', + 'type' => 'leads', + 'id' => $lead->getId() + ]; + + $response = $this->elastic->get($params); + + $this->assertTrue($response['_source']['id'] == $lead->getId()); + } + + /** + * Delete from a record. + * + * @return void + */ + public function testDeleteOneDocumentFromRecordModel() + { + //cli need the config + $this->config = $this->getDI()->getConfig(); + $this->elastic = $this->getDI()->getElastic(); + + $lead = Leads::findFirst(); + + // Get elasticsearch class handler instance + $elasticsearch = new IndexBuilder(); + + //insert into elastic + $result = $elasticsearch->deleteDocument($lead); //depth + + $this->assertTrue($result['_shards']['successful'] == 1); + } +} diff --git a/tests/unit/IndicesTest.php b/tests/unit/IndicesTest.php index 8895fea..c9c9a56 100755 --- a/tests/unit/IndicesTest.php +++ b/tests/unit/IndicesTest.php @@ -71,15 +71,5 @@ public function testDeletetDocumentToIndex() $elasticsearch::deleteDocument($indices); } - /** - * this runs before everyone - */ - protected function setUp() - { - $this->_getDI(); - } - - protected function tearDown() - { - } + } diff --git a/tests/unit/PhalconUnitTestCase.php b/tests/unit/PhalconUnitTestCase.php index 68c43d3..828dcaf 100755 --- a/tests/unit/PhalconUnitTestCase.php +++ b/tests/unit/PhalconUnitTestCase.php @@ -4,6 +4,8 @@ use \Phalcon\Di; use \Phalcon\Test\UnitTestCase as PhalconTestCase; use Phalcon\Annotations\Adapter\Memcached; +use Elasticsearch\ClientBuilder; +use Baka\Auth\Models\Apps; abstract class PhalconUnitTestCase extends PhalconTestCase { @@ -23,7 +25,7 @@ abstract class PhalconUnitTestCase extends PhalconTestCase private $_loaded = false; /** - * Setup phalconPHP DI to use for testing components + * Setup phalconPHP DI to use for testing components. * * @return Phalcon\DI */ @@ -34,16 +36,16 @@ protected function _getDI() $di = new Phalcon\DI(); /** - * DB Config + * DB Config. * @var array */ $this->_config = new \Phalcon\Config([ 'database' => [ 'adapter' => 'Mysql', - 'host' => getenv('DATABASE_HOST'), - 'username' => getenv('DATABASE_USER'), - 'password' => getenv('DATABASE_PASS'), - 'dbname' => getenv('DATABASE_NAME'), + 'host' => getenv('DATA_API_MYSQL_HOST'), + 'username' => getenv('DATA_API_MYSQL_USER'), + 'password' => getenv('DATA_API_MYSQL_PASS'), + 'dbname' => getenv('DATA_API_MYSQL_NAME'), ], 'memcache' => [ 'host' => getenv('MEMCACHE_HOST'), @@ -51,9 +53,9 @@ protected function _getDI() ], 'namespace' => [ 'controller' => '', - 'models' => '', + 'models' => 'Test\Model', 'library' => '', - 'elasticIndex' => '', + 'elasticIndex' => 'Test\Indices', ], 'email' => [ 'driver' => 'smtp', @@ -94,7 +96,7 @@ protected function _getDI() }); /** - * Everything needed initialize phalconphp db + * Everything needed initialize phalconphp db. */ $di->set('mail', function () use ($config, $di) { @@ -105,7 +107,7 @@ protected function _getDI() }); /** - * config queue by default Beanstalkd + * config queue by default Beanstalkd. */ $di->set('queue', function () use ($config) { //Connect to the queue @@ -142,6 +144,16 @@ protected function _getDI() return $view; }); + $di->set('elastic', function () use ($config) { + $hosts = $config->elasticSearch->hosts->toArray(); + + $client = ClientBuilder::create() + ->setHosts($hosts) + ->build(); + + return $client; + }); + $di->set('modelsManager', function () { return new Phalcon\Mvc\Model\Manager(); }, true); @@ -163,8 +175,12 @@ protected function _getDI() return $connection; }); + $di->set('app', function () { + return Apps::findFirst(); + }, true); + /** - * Start the session the first time some component request the session service + * Start the session the first time some component request the session service. */ $di->set('session', function () use ($config) { $memcache = new \Phalcon\Session\Adapter\Memcache([ @@ -198,4 +214,12 @@ protected function _getDI() return $di; } + + /** + * this runs before everyone + */ + protected function setUp() + { + $this->_getDI(); + } }