Elasticsearch için kullanılacak olan bir symfony uygulama temeli yaratmak - versiyon 2

12/03/2016 - ELASTICSEARCH, SYMFONY

Elasticsearch ile sorgu yapmak için aşağıdaki symfony uygulama temelini kullanabilirsiniz. Bu örnek FOSElasticaBundle paketini kullanıyor.


Kurulum


Öncelikle composer ile friendsofsymfony/elastica-bundle paketini kurun. Bu yazıyı yazarken friendsofsymfony/elastica-bundle paketi sadece elasticsearch 1.7.4 ve alt versiyonlarını destekliyordu.


Config.yml


Bu sadece bir örnek index. Aşağıdaki ayarlara bakarsanız, aynı sorgu içinde sadece "title" ve "description" alanları üzerinde hem full-text hem de sıralama yapabilirsiniz. Daha fazla bilgi için String Sorting and Multifields sayfasını okuyun.


fos_elastica:
    clients:
        default: { host: 127.0.0.1, port: 9203 }
    indexes:
        post_index:
            client: default
            index_name: post_%kernel.environment%
            types:
                post:
                    mappings:
                        id:
                            type: integer
                            index: not_analyzed
                        title:
                            type: string
                            analyzer: english
                            fields:
                                raw:
                                    type: string
                                    index: not_analyzed
                        description:
                            type: string
                            analyzer: english
                            fields:
                                raw:
                                    type: string
                                    index: not_analyzed
                        year:
                            type: integer
                            index: not_analyzed
                        price:
                            type: double
                            index: not_analyzed
                        is_published:
                            type: boolean
                            index: not_analyzed
                        created_at:
                            type: date
                            index: not_analyzed
                    persistence:
                        driver: orm
                        model: Application\SearchBundle\Entity\Post
                        finder: ~
                        provider: ~
                        listener: ~

Post entity


Yukarıda gördüğümüz gibi "author" alanını index içinde kullanmadık, bu nedenle elasticsearch içinde o alanı kullanamayız.


namespace Application\SearchBundle\Entity;

use DateTime;
use Doctrine\ORM\Mapping as ORM;

/**
 * @ORM\Entity
 * @ORM\Table(name="post")
 * @ORM\HasLifecycleCallbacks
 */
class Post
{
    /**
     * @ORM\Id
     * @ORM\Column(type="integer")
     * @ORM\GeneratedValue(strategy="AUTO")
     */
    private $id;

    /**
     * @ORM\Column(name="title", type="string", length=100)
     */
    private $title;

    /**
     * @ORM\Column(name="description", type="text")
     */
    private $description;

    /**
     * @ORM\Column(name="author", type="string", length=100)
     */
    private $author;

    /**
     * @ORM\Column(name="year", type="string", length=4)
     */
    private $year;

    /**
     * @ORM\Column(name="price", type="decimal", precision=4, scale=2)
     */
    private $price;

    /**
     * @ORM\Column(name="is_published", type="boolean")
     */
    private $isPublished;

    /**
     * @ORM\Column(name="created_at", type="datetime")
     */
    private $createdAt;
}

SearchController


Aşağıda gördüğümüz gibi, arama yapabileceğimiz örnek bir URL GET /search/elasticsearch?page=1&limit=10&sort=title,-description&fields=title,description&keywords=hello%20world gibi olacak.


namespace Application\SearchBundle\Controller;

use Application\SearchBundle\Service\PostElasticsearchServiceInterface;
use Symfony\Bundle\FrameworkBundle\Controller\Controller;
use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route;
use Sensio\Bundle\FrameworkExtraBundle\Configuration\Method;
use Sensio\Bundle\FrameworkExtraBundle\Configuration\Template;
use Symfony\Component\HttpFoundation\Request;

/**
 * @Route("/search", service="application_search.controller.search")
 */
class SearchController extends Controller
{
    private $postElasticsearchService;

    public function __construct(
        PostElasticsearchServiceInterface $postElasticsearchService
    ) {
        $this->postElasticsearchService = $postElasticsearchService;
    }

    /**
     * @param Request $request
     *
     * @Method({"GET"})
     * @Route("/elasticsearch")
     * @Template()
     *
     * @return array
     */
    public function searchElasticsearchAction(Request $request)
    {
        $page = $request->query->get('page', 1);
        $limit = $request->query->get('limit', 10);
        $sort = $request->query->get('sort');
        $fields = $request->query->get('fields');
        $keywords = $request->query->get('keywords');

        $posts = $this->postElasticsearchService->getPosts($page, $limit, $sort, $fields, $keywords);

        return [
            'page' => $page,
            'limit' => $limit,
            'sort' => $sort,
            'fields' => $fields,
            'keywords' => $keywords,
            'posts' => $posts
        ];
    }
}

Controllers.yml


services:
    application_search.controller.search:
        class: Application\SearchBundle\Controller\SearchController
        arguments:
            - @application_search.service.post_elasticsearch

PostElasticsearchServiceInterface


namespace Application\SearchBundle\Service;

interface PostElasticsearchServiceInterface
{
    public function getPosts($page, $limit, $sort = null, $fields = null, $keywords = null);
}

PostElasticsearchService


namespace Application\SearchBundle\Service;

use Application\SearchBundle\Factory\PostFactoryInterface;
use Application\SearchBundle\Util\Elasticsearch;

class PostElasticsearchService implements PostElasticsearchServiceInterface
{
    private $elasticsearch;
    private $postFactory;

    public function __construct(
        Elasticsearch $elasticsearch,
        PostFactoryInterface $postFactory
    ) {
        $this->elasticsearch = $elasticsearch;
        $this->postFactory = $postFactory;
    }

    public function getPosts($page, $limit, $sort = null, $fields = null, $keywords = null)
    {
        return $this->postFactory->createPostResult(
            $this->elasticsearch->find($page, $limit, $sort, $fields, $keywords)
        );
    }
}

Services.yml


services:
    application_search.service.post_elasticsearch:
        class: Application\SearchBundle\Service\PostElasticsearchService
        arguments:
            - @application_search.util.elasticsearch
            - @application_search.factory.post

PostFactoryInterface


namespace Application\SearchBundle\Factory;

interface PostFactoryInterface
{
    public function createPostResult(array $result = []);
}

PostFactory


namespace Application\SearchBundle\Factory;

use Application\SearchBundle\Model\Post;
use Elastica\Result as ElasticaResult;
use Application\SearchBundle\Model\Result;

class PostFactory implements PostFactoryInterface
{
    public function createPostResult(array $result = [])
    {
        $postResult = new Result();
        $postResult->page = $result['page'];
        $postResult->limit = $result['limit'];
        $postResult->total = $result['total'];
        $postResult->posts = $this->getPosts($result['data']);

        return $postResult;
    }

    private function getPosts(array $posts = [])
    {
        $data = [];

        /** @var ElasticaResult $result */
        foreach ($posts as $result) {
            $post = new Post();
            $post->id = $result->getSource()['id'];
            $post->title = $result->getSource()['title'];
            $post->description = $result->getSource()['description'];
            $post->year = $result->getSource()['year'];
            $post->price = $result->getSource()['price'];
            $post->isPublished = $result->getSource()['is_published'];
            $post->createdAt = $result->getSource()['created_at'];

            $data[] = $post;
        }

        return $data;
    }
}

Factories.yml


services:
    application_search.factory.post:
        class: Application\SearchBundle\Factory\PostFactory

Elasticsearch utility class


Eğer daha farklı sorgu yapma gibi bir planınız olursa, getSearchByQuery() methodunu değiştirmeniz yeterli olur. Şu an için sadece basit bir full-text arama yapmaya ayarlıdır.


namespace Application\SearchBundle\Util;

use Elastica\Query;
use Elastica\Type;
use Symfony\Component\HttpKernel\Exception\BadRequestHttpException;

class Elasticsearch
{
    const ANALYSED_SUFFIX = '.raw';

    // These fields must be set as "multivalue" fields in config.yml
    private $stringSortingMultivalueFields = ['title', 'description'];
    private $postType;

    public function __construct(Type $postType)
    {
        $this->postType = $postType;
    }

    public function find($page, $limit, $sort = null, $fields = null, $keywords = null)
    {
        $page = $page > 0 ? $page : 1;
        $query = null;

        if (!$fields && !$keywords) {
            $query = $this->getListByQuery($sort);
        } elseif ($fields && $keywords) {
            $query = $this->getSearchByQuery($sort, $fields, $keywords);
        }

        if (!$query) {
            throw new BadRequestHttpException('Invalid request.');
        }

        $queryObject = (new Query($query))
            ->setSize($limit)
            ->setFrom($page-1);

        $result = $this->postType->search($queryObject);

        return [
            'page' => $page,
            'limit' => $limit,
            'total' => $result->getTotalHits(),
            'data' => $result->getResults()
        ];
    }

    private function getListByQuery($sort = null)
    {
        $query = [];

        $query['query']['bool']['must'][]['match_all'] = [];
        $query['sort'] = $this->getSort($sort);

        return $query;
    }

    private function getSearchByQuery($sort, $fields, $keywords)
    {
        $query = [];

        $searchFields = explode(',', $fields);
        if (count($searchFields) == 1) {
            $query['query']['match'][$searchFields[0]]['query'] = $keywords;
        } else {
            $query['query']['multi_match']['type'] = 'most_fields';
            $query['query']['multi_match']['query'] = $keywords;
            $query['query']['multi_match']['fields'] = $searchFields;
        }
        $query['sort'] = $this->getSort($sort, true);

        return $query;
    }

    private function getSort($sort = null, $search = false)
    {
        $id = true;
        $query = [];

        // Applies to only "search" requests
        if ($search) {
            $query[]['_score']['order'] = 'desc';
        }

        if (!is_null($sort)) {
            $sortFields = explode(',', $sort);
            foreach ($sortFields as $sortField) {
                // If the field is one of string sorting ones then use not_analyzed "*.raw" version
                if (in_array($sortField, $this->stringSortingMultivalueFields)) {
                    $sortField = $sortField.self::ANALYSED_SUFFIX;
                }

                // Get correct direction
                if (substr($sortField, 0, 1) === '-') {
                    $sortField = substr($sortField, 1);
                    $query[][$sortField]['order'] = 'desc';
                } else {
                    $query[][$sortField]['order'] = 'asc';
                }

                if ($sortField == 'id') {
                    $id = false;
                }
            }

            // If "id" field is not specified by user in request then include it by default
            if ($id) {
                $query[]['id']['order'] = 'asc';
            }
        } else {
            $query[]['id']['order'] = 'asc';
        }

        return $query;
    }
}

Utils.yml


Elasticsearch servis ismi config.yml içindeki bilgilere göre otomatik olarak yaratılır yani, @fos_elastica.index.indexes_name.type_name bizim için @fos_elastica.index.post_index.post olacaktır.


services:
    application_search.util.elasticsearch:
        class: Application\SearchBundle\Util\Elasticsearch
        arguments:
            - @fos_elastica.index.post_index.post

Result model class


namespace Application\SearchBundle\Model;

class Result
{
    /**
     * @var int
     */
    public $page;

    /**
     * @var int
     */
    public $limit;

    /**
     * @var int
     */
    public $total;

    /**
     * @var Post[]
     */
    public $posts = [];
}

Post model class


namespace Application\SearchBundle\Model;

class Post
{
    /**
     * @var int
     */
    public $id;

    /**
     * @var string
     */
    public $title;

    /**
     * @var string
     */
    public $description;

    /**
     * @var string
     */
    public $author;

    /**
     * @var int
     */
    public $year;

    /**
     * @var double
     */
    public $price;

    /**
     * @var boolean
     */
    public $isPublished;

    /**
     * @var string
     */
    public $createdAt;
}

PopulatePostCommand


Öncelikle veritabanını doldurmak için $ app/console populate:post --total=5000 komutunu kullanın.


namespace Application\SearchBundle\Command;

use Application\SearchBundle\Entity\Post;
use Doctrine\ORM\EntityManagerInterface;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;

class PopulatePostCommand extends Command
{
    const TOTAL = 'total';

    private $entityManager;
    private $authors = ['Robert DeNiro', 'Robert', 'DeNiro', 'Al Pacino', 'Al', 'Pacino', 'Andy Garcia', 'Andy'];
    private $titles = ['Title', 'Title 1', 'Title 2', 'Title 3', 'Eltit', 'Eltit A', 'Eltit B', 'Desc', 'Desc 3'];
    private $descriptions = ['Desc', 'Desc 1', 'Desc 2', 'Cript', 'Cript A', 'Cript B', 'Title', 'Title 3'];
    private $years = [2000, 2005, 2010, 2015];
    private $prices = [0.5, 1, 2.5, 3.99, 4.00, 5.55];

    public function __construct(EntityManagerInterface $entityManager)
    {
        parent::__construct();

        $this->entityManager = $entityManager;
    }

    protected function configure()
    {
        $this
            ->setName('populate:post')
            ->setDescription('Populates post entity.')
            ->addOption(
                self::TOTAL,
                null,
                InputOption::VALUE_REQUIRED,
                'Defines total amount of records to be created.'
            );
    }

    protected function execute(InputInterface $input, OutputInterface $output)
    {
        for ($i = 0; $i < $input->getOption(self::TOTAL); $i++) {
            $post = new Post();
            $post->setAuthor($this->authors[array_rand($this->authors)]);
            $post->setTitle($this->titles[array_rand($this->titles)]);
            $post->setDescription($this->descriptions[array_rand($this->descriptions)]);
            $post->setYear($this->years[array_rand($this->years)]);
            $post->setPrice($this->prices[array_rand($this->prices)]);
            $post->setIsPublished(array_rand([true, false]));

            $this->entityManager->persist($post);
            if ($i%10 == 0) {
                $this->entityManager->flush();
            }
        }
        $this->entityManager->flush();
    }
}

Commands.yml


services:
    application_search.command.populate_post:
        class: Application\SearchBundle\Command\PopulatePostCommand
        tags:
            - { name: console.command }
        arguments:
            - @doctrine.orm.entity_manager

Elasticsearch'ü doldurmak


$ app/console fos:elastica:populate
 5000/5000 [============================] 100%
Populating post_index/post
Refreshing post_index
Refreshing post_index

Testler


Sıralama yapmadan sadece listeleme


# Request
GET /search/elasticsearch?page=1&limit=10

# Response array
Array
(
    [query] => Array
        (
            [bool] => Array
                (
                    [must] => Array
                        (
                            [0] => Array
                                (
                                    [match_all] => Array
                                        (
                                        )
                                )
                        )
                )
        )
    [sort] => Array
        (
            [0] => Array
                (
                    [id] => Array
                        (
                            [order] => asc
                        )
                )
        )
    [from] => 0
    [size] => 10
)

# Response json
{
  "query": {
    "bool": {
      "must": [
        {
          "match_all": [
            
          ]
        }
      ]
    }
  },
  "sort": [
    {
      "id": {
        "order": "asc"
      }
    }
  ],
  "from": 0,
  "size": "10"
}

Sıralama yaparak sadece listeleme


# Request
GET /search/elasticsearch?page=1&limit=10&sort=title,-year

# Response array
Array
(
    [query] => Array
        (
            [bool] => Array
                (
                    [must] => Array
                        (
                            [0] => Array
                                (
                                    [match_all] => Array
                                        (
                                        )
                                )
                        )
                )
        )
    [sort] => Array
        (
            [0] => Array
                (
                    [title.raw] => Array
                        (
                            [order] => asc
                        )
                )
            [1] => Array
                (
                    [year] => Array
                        (
                            [order] => desc
                        )
                )
            [2] => Array
                (
                    [id] => Array
                        (
                            [order] => asc
                        )
                )
        )
    [from] => 0
    [size] => 10
)

# Response json
{
  "query": {
    "bool": {
      "must": [
        {
          "match_all": [
            
          ]
        }
      ]
    }
  },
  "sort": [
    {
      "title.raw": {
        "order": "asc"
      }
    },
    {
      "year": {
        "order": "desc"
      }
    },
    {
      "id": {
        "order": "asc"
      }
    }
  ],
  "from": 0,
  "size": "10"
}

Tek alan üzerinden arama


# Request
GET /search/elasticsearch?page=1&limit=10&sort=description&fields=title&keywords=inanzzz

# Response array
Array
(
    [query] => Array
        (
            [match] => Array
                (
                    [title] => Array
                        (
                            [query] => inanzzz
                        )
                )
        )
    [sort] => Array
        (
            [0] => Array
                (
                    [_score] => Array
                        (
                            [order] => desc
                        )
                )
            [1] => Array
                (
                    [description.raw] => Array
                        (
                            [order] => asc
                        )
                )
            [2] => Array
                (
                    [id] => Array
                        (
                            [order] => asc
                        )
                )
        )
    [from] => 0
    [size] => 10
)

# Response json
{
  "query": {
    "match": {
      "title": {
        "query": "inanzzz"
      }
    }
  },
  "sort": [
    {
      "_score": {
        "order": "desc"
      }
    },
    {
      "description.raw": {
        "order": "asc"
      }
    },
    {
      "id": {
        "order": "asc"
      }
    }
  ],
  "from": 0,
  "size": "10"
}

Birden fazla alan üzerinden arama


# Request
GET /search/elasticsearch?page=1&limit=10&sort=description&fields=title,description&keywords=inanzzz

# Response array
Array
(
    [query] => Array
        (
            [multi_match] => Array
                (
                    [type] => most_fields
                    [query] => inanzzz
                    [fields] => Array
                        (
                            [0] => title
                            [1] => description
                        )
                )
        )
    [sort] => Array
        (
            [0] => Array
                (
                    [_score] => Array
                        (
                            [order] => desc
                        )
                )
            [1] => Array
                (
                    [description.raw] => Array
                        (
                            [order] => asc
                        )
                )
            [2] => Array
                (
                    [id] => Array
                        (
                            [order] => asc
                        )
                )
        )
    [from] => 0
    [size] => 10
)

# Response json
{
  "query": {
    "multi_match": {
      "type": "most_fields",
      "query": "inanzzz",
      "fields": [
        "title",
        "description"
      ]
    }
  },
  "sort": [
    {
      "_score": {
        "order": "desc"
      }
    },
    {
      "description.raw": {
        "order": "asc"
      }
    },
    {
      "id": {
        "order": "asc"
      }
    }
  ],
  "from": 0,
  "size": "10"
}