Хотя вопрос старый, я отвечаю как
Недавно я задал аналогичный вопрос (без ответов или комментариев через несколько дней), в котором я разобрался и который имеет отношение к этому вопросу.
Solr сильно изменился за эти годы, и существующая документация (там, где она существует) по этой теме одновременно запутана и иногда ошибочна.
Несмотря на то, что этот ответ длинный, он дает решение вопроса с примером и документацией.
Вкратце, мой уже удаленный вопрос StackOverflow заключался в извлечении пользовательского (например, ‹my_id›‹/my_id) текста с тегами из HTML с использованием Apache Solr. Вспомогательной задачей было то, как индексировать HTML-страницы, включая пользовательские элементы HTML: атрибуты.
Краткий ответ заключается в том, что индексировать стандартные элементы HTML (a; div; h1; h2; li; meta; p; title; ... https://www.w3.org/)относительнолегко.TR/2005/WD-xhtml2-20050527/elements.html), сложно включить пользовательские наборы тегов без жесткого использования правильно отформатированных XML-файлов и функций обновления в Solr (см., например: https://lucene.apache.org/solr/guide/6_6/uploading-data-with-index-handlers.html#uploading-data-with-index-handlers), или использование параметра captureAttr
с Apache Tika, встроенного в Solr через ExtractingRequestHandler
(описано ниже) или других инструментов, таких как Apache Nutch.
Стандартные элементы HTML, такие как <title>Solr HTML Indexing Tests</title>
, легко индексируются; однако нестандартные элементы, такие как <my_id>bt-ic8eew2u</my_id>
, игнорируются.
Хотя вы можете применять решения на основе XML, такие как <field name="my_id">bt-ic8eew2u</field>
, я предпочитаю простое решение на основе HTML — отсюда и подход с метаданными HTML.
Окружающая среда: командная строка Arch Linux (x86_64); Apache Solr 8.7.0; Интерфейс администратора Solr (http://localhost:8983/solr/#/gettingstarted/query) в FireFox 83.0
Тестовый файл (solr_test9.html):
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" lang="en-us">
<head>
<meta charset="UTF-8" />
<title>Solr HTML Indexing Tests</title>
<meta name="date_created" content="2019-11-01" />
<meta name="source_url" content="/mnt/Vancouver/programming/datasci/solr/test/solr_test9.html" />
<!-- <my_id>bt-ic8eew2u</my_id> -->
<meta name="doc_id" content="bt-ic8eeW2U" />
<meta name="date_pub" content="2020-11-16" />
</head>
<body>
<h1>Apples</h1>
<p>I like apples.</p>
<h2>Bananas</h2>
<p>I also like bananas.</p>
<p><div id="div1">This text is located in div element 1.</div></p>
<p><div id="div2">This text is located in div element 2.</div></p>
<br/>
Lorem ipsum dolor sit amet, consectetur adipiscing elit.
<br/>
<p>Suspendisse efficitur pulvinar elementum.</p>
<p>My website is <a href="https://buriedtruth.com/">BuriedTruth.com</a>.</p>
<h1>Nova Scotia</h1>
<p>Nova Scotia is a province on the east coast of Canada.</p>
<h2>Capital of Nova Scotia</h2>
<p>Halifax is the capital of N.S.</p>
<p>Halifax is also N.S.'s largest city.</p>
<h1>British Columbia</h1>
<h2>Capital of British Columbia</h2>
<p>Victoria is the capital of B.C.</p>
<p>Vancouver is the largest city in B.C., however.</p>
<p>Non-terminated sentence (missing period)</p>
<meta name="date_current" content="2020-11-17" />
<!-- Comments like these are not indexed. -->
<p>Current date: 2020-11-17</p>
</body>
</html>
solrconfig.xml
Вот соответствующие дополнения к моему файлу solrconfig.xml
.
<!-- SOLR CELL PLUGINS: -->
<lib dir="${solr.install.dir:../../..}/contrib/extraction/lib" regex=".*\.jar" />
<lib dir="${solr.install.dir:../../..}/dist/" regex="solr-cell-\d.*\.jar" />
<!-- https://lucene.472066.n3.nabble.com/Prons-an-Cons-of-Startup-Lazy-a-Handler-td4059111.html -->
<requestHandler name="/update/extract"
class="org.apache.solr.handler.extraction.ExtractingRequestHandler">
<lst name="defaults">
<str name="lowernames">true</str>
<str name="uprefix">ignored_</str>
<str name="capture">div</str>
<str name="fmap.div">div</str>
<str name="capture">h1</str>
<str name="fmap.h1">h1</str>
<str name="capture">h2</str>
<str name="fmap.h2">h2_t</str>
<str name="capture">p</str>
<!-- <str name="fmap.p">p_t</str> -->
<str name="fmap.p">p</str>
<!-- COMMENT: note that the entries above refer to standard -->
<!-- HTML elements. As long as you have <meta/> (metadata) -->
<!-- entries ("doc-id", "date_pub" ...) in your schema then -->
<!-- Solr will automatically pick them up when indexing ... -->
<!-- (hence no need to include those, here!). -->
</lst>
</requestHandler>
<!-- https://doc.lucidworks.com/fusion-server/5.2/reference/solr-reference-guide/7.7.2/update-request-processors.html -->
<!-- The update.autoCreateFields property can be turned to false to disable schemaless mode -->
<updateRequestProcessorChain name="add-unknown-fields-to-the-schema" default="${update.autoCreateFields:true}"
processor="uuid,remove-blank,field-name-mutating,parse-boolean,parse-long,parse-double,parse-date,add-schema-fields">
<processor class="solr.LogUpdateProcessorFactory"/>
<processor class="solr.DistributedUpdateProcessorFactory"/>
<!-- ======================================== -->
<!-- https://lucene.apache.org/solr/7_4_0/solr-core/org/apache/solr/update/processor/RegexReplaceProcessorFactory.html -->
<processor class="solr.RegexReplaceProcessorFactory">
<str name="fieldName">content</str>
<str name="fieldName">title</str>
<str name="fieldName">p</str>
<!-- Case-sensitive (and only one pattern:replacement allowed, so use as many copies): -->
<!-- of this processor as needed: -->
<str name="pattern">\s+</str>
<str name="replacement"> </str>
<bool name="literalReplacement">true</bool>
</processor>
<!-- Solr bug? URLs parse as "rect https..." Managed-schema (Admin UI): defined p as text_general -->
<!-- but did not parse. Looking at content | title: text_general copied to string, so added -->
<!-- copyfield of p (text_general) as p_str ... regex below now works! -->
<!-- https://stackoverflow.com/questions/22178700/solr-extractingrequesthandler-extracting-rect-in-links/64882751#64882751 -->
<processor class="solr.RegexReplaceProcessorFactory">
<str name="fieldName">content</str>
<str name="fieldName">title</str>
<str name="fieldName">p</str>
<!-- Case-sensitive (and only one pattern:replacement allowed, so use as many copies): -->
<!-- of this processor as needed: -->
<str name="pattern">rect http</str>
<str name="replacement">http</str>
<bool name="literalReplacement">true</bool>
</processor>
<!-- ======================================== -->
<!-- This needs to be last (may need to clear documents and re-index to see changes, e.g. Solr Admin UI): -->
<processor class="solr.RunUpdateProcessorFactory"/>
</updateRequestProcessorChain>
управляемая схема (schema.xml):
Я редактировал схему Solr через пользовательский интерфейс администратора. По сути, для любых метаданных HTML, которые вы хотите индексировать, добавьте поле с аналогичным названием (соответствующего типа: например, text_general
| string
| pdate
| ...).
Например, чтобы получить метаданные doc-id и date_pub, я создал следующие (соответствующие) записи схемы:
<field name="doc_id" type="string" uninvertible="true" indexed="true" stored="true"/>
<field name="date_pub" type="pdate" uninvertible="true" indexed="true" stored="true"/>
индексирование
Вот как я проиндексировал этот тестовый HTML-файл,
[victoria@victoria solr-8.7.0]$ date; pwd; ls -l; echo; ls -l server/solr/gettingstarted/conf/
Tue Nov 17 02:18:12 PM PST 2020
/mnt/Vancouver/apps/solr/solr-8.7.0
total 1792
drwxr-xr-x 3 victoria victoria 4096 Nov 17 13:26 bin
-rw-r--r-- 1 victoria victoria 946955 Oct 28 02:40 CHANGES.txt
drwxr-xr-x 12 victoria victoria 4096 Oct 29 07:09 contrib
drwxr-xr-x 4 victoria victoria 4096 Nov 15 12:33 dist
drwxr-xr-x 3 victoria victoria 4096 Nov 15 12:33 docs
drwxr-xr-x 6 victoria victoria 4096 Oct 28 02:40 example
drwxr-xr-x 2 victoria victoria 36864 Oct 28 02:40 licenses
-rw-r--r-- 1 victoria victoria 12646 Oct 28 02:21 LICENSE.txt
-rw-r--r-- 1 victoria victoria 766662 Oct 28 02:40 LUCENE_CHANGES.txt
-rw-r--r-- 1 victoria victoria 27540 Oct 28 02:21 NOTICE.txt
-rw-r--r-- 1 victoria victoria 7490 Oct 28 02:40 README.txt
drwxr-xr-x 11 victoria victoria 4096 Nov 15 12:40 server
total 208
drwxr-xr-x 2 victoria victoria 4096 Oct 28 02:21 lang
-rw-r--r-- 1 victoria victoria 33888 Nov 17 13:20 managed-schema
-rw-r--r-- 1 victoria victoria 873 Oct 28 02:21 protwords.txt
-rw-r--r-- 1 victoria victoria 33788 Nov 17 11:36 schema.xml.2020-11-17.13:01
-rw-r--r-- 1 victoria victoria 59248 Nov 17 13:16 solrconfig.xml
-rw-r--r-- 1 victoria victoria 59151 Nov 17 12:59 solrconfig.xml.2020-11-17.13:01
-rw-r--r-- 1 victoria victoria 781 Oct 28 02:21 stopwords.txt
-rw-r--r-- 1 victoria victoria 1124 Oct 28 02:21 synonyms.txt
[victoria@victoria solr-8.7.0]$ solr restart; sleep 1; post -c gettingstarted /mnt/Vancouver/programming/datasci/solr/test/solr_test9.html
Sending stop command to Solr running on port 8983 ... waiting up to 180 seconds to allow Jetty process 3511453 to stop gracefully.
Waiting up to 180 seconds to see Solr running on port 8983 [|]
Started Solr server on port 8983 (pid=3572520). Happy searching!
/usr/lib/jvm/java-8-openjdk/jre//bin/java -classpath /mnt/Vancouver/apps/solr/solr-8.7.0/dist/solr-core-8.7.0.jar -Dauto=yes -Dc=gettingstarted -Ddata=files org.apache.solr.util.SimplePostTool /mnt/Vancouver/programming/datasci/solr/test/solr_test9.html
SimplePostTool version 5.0.0
Posting files to [base] url http://localhost:8983/solr/gettingstarted/update...
Entering auto mode. File endings considered are xml,json,jsonl,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log
POSTing file solr_test9.html (text/html) to [base]/extract
1 files indexed.
COMMITting Solr index changes to http://localhost:8983/solr/gettingstarted/update...
Time spent: 0:00:00.755
[victoria@victoria solr-8.7.0]$
... и вот результат (интерфейс администратора Solr: http://localhost:8983/solr/#/gettingstarted/query)
http://localhost:8983/solr/gettingstarted/select?q=*%3A*
{
"responseHeader":{
"status":0,
"QTime":0,
"params":{
"q":"*:*",
"_":"1605651674401"}},
"response":{"numFound":1,"start":0,"numFoundExact":true,"docs":[
{
"id":"/mnt/Vancouver/programming/datasci/solr/test/solr_test9.html",
"stream_size":[1428],
"x_parsed_by":["org.apache.tika.parser.DefaultParser",
"org.apache.tika.parser.html.HtmlParser"],
"stream_content_type":["text/html"],
"date_created":"2019-11-01T00:00:00Z",
"date_current":["2020-11-17"],
"resourcename":["/mnt/Vancouver/programming/datasci/solr/test/solr_test9.html"],
"title":["Solr HTML Indexing Tests"],
"date_pub":"2020-11-16T00:00:00Z",
"doc_id":"bt-ic8eeW2U",
"source_url":"/mnt/Vancouver/programming/datasci/solr/test/solr_test9.html",
"dc_title":["Solr HTML Indexing Tests"],
"content_encoding":["UTF-8"],
"content_type":["application/xhtml+xml; charset=UTF-8"],
"content":[" en-us stream_size 1428 X-Parsed-By org.apache.tika.parser.DefaultParser X-Parsed-By org.apache.tika.parser.html.HtmlParser stream_content_type text/html date_created 2019-11-01 resourceName /mnt/Vancouver/programming/datasci/solr/test/solr_test9.html date_pub 2020-11-16 doc_id bt-ic8eeW2U source_url /mnt/Vancouver/programming/datasci/solr/test/solr_test9.html dc:title Solr HTML Indexing Tests Content-Encoding UTF-8 Content-Language en-us Content-Type application/xhtml+xml; charset=UTF-8 Solr HTML Indexing Tests Lorem ipsum dolor sit amet, consectetur adipiscing elit. "],
"div":[" div1 This text is located in div element 1. div2 This text is located in div element 2."],
"p":[" I like apples. I also like bananas. Suspendisse efficitur pulvinar elementum. My website is https://buriedtruth.com/ BuriedTruth.com . Nova Scotia is a province on the east coast of Canada. Halifax is the capital of N.S. Halifax is also N.S.'s largest city. Victoria is the capital of B.C. Vancouver is the largest city in B.C., however. Non-terminated sentence (missing period) Current date: 2020-11-17"],
"h1":[" Apples Nova Scotia British Columbia"],
"h2_t":" Bananas Capital of Nova Scotia Capital of British Columbia",
"_version_":1683647678197530624}]
}}
ОБНОВЛЕНИЕ -- managed-schema
›› schema.xml
особенности:
Хотя это и не связано с исходным вопросом, следующее содержание связано с моим ответом (выше) - в частности, с особенностями, связанными с переключением с managed-schema
Solr на классический (управляемый пользователем) schema.xml
. Он включен здесь, чтобы предоставить полное решение.
Во-первых, добавьте
<schemaFactory class="ClassicIndexSchemaFactory"/>
в ваш файл solrconfig.xml
.
Затем отредактируйте это: --›
<updateRequestProcessorChain
name="add-unknown-fields-to-the-schema"
default="${update.autoCreateFields:true}"
processor="uuid,remove-blank,field-name-mutating,parse-boolean,
parse-long,parse-double,parse-date,add-schema-fields">
... к этому:
<updateRequestProcessorChain
processor="uuid,remove-blank,field-name-mutating,parse-boolean,
parse-long,parse-double,parse-date">
то есть удалить
name="add-unknown-fields-to-the-schema"
default="${update.autoCreateFields:true}"
add-schema-fields
Переименуйте managed-schema
в schema.xml
и перезапустите Solr или перезагрузите ядро, чтобы изменения вступили в силу.
Чтобы еще больше расширить мой пример (выше), вот образец <updateRequestProcessorChain />
и вывод в HTML-коде, который я также предоставил (выше).
solrconfig.xml (часть):
<updateRequestProcessorChain
processor="uuid,remove-blank,field-name-mutating,parse-boolean,parse-long,parse-double,parse-date">
<processor class="solr.LogUpdateProcessorFactory"/>
<processor class="solr.DistributedUpdateProcessorFactory"/>
<processor class="solr.RegexReplaceProcessorFactory">
<str name="fieldName">content</str>
<str name="fieldName">title</str>
<str name="fieldName">p</str>
<!-- Case-sensitive (and only one pattern:replacement allowed, so use as many copies): -->
<!-- of this processor as needed: -->
<str name="pattern">\s+</str>
<str name="replacement"> </str>
<bool name="literalReplacement">true</bool>
</processor>
<processor class="solr.RegexReplaceProcessorFactory">
<str name="fieldName">content</str>
<str name="fieldName">title</str>
<str name="fieldName">p</str>
<!-- Case-sensitive (and only one pattern:replacement allowed, so use as many copies): -->
<!-- of this processor as needed: -->
<str name="pattern">rect http</str>
<str name="replacement">http</str>
<bool name="literalReplacement">true</bool>
</processor>
<processor class="solr.RegexReplaceProcessorFactory">
<str name="fieldName">content</str>
<str name="fieldName">title</str>
<str name="pattern">[sS]olr</str>
<str name="replacement">APPLE</str>
<bool name="literalReplacement">true</bool>
</processor>
<processor class="solr.RegexReplaceProcessorFactory">
<str name="fieldName">content</str>
<str name="fieldName">title</str>
<str name="pattern">HTML</str>
<str name="replacement">BANANA</str>
<bool name="literalReplacement">true</bool>
</processor>
<processor class="solr.RunUpdateProcessorFactory"/>
</updateRequestProcessorChain>
вывод
{
"responseHeader":{
"status":0,
"QTime":32,
"params":{
"q":"*:*",
"_":"1605767164812"}},
"response":{"numFound":1,"start":0,"numFoundExact":true,"docs":[
{
"id":"/mnt/Vancouver/programming/datasci/solr/test/solr_test9.html",
"stream_size":[1628],
"x_parsed_by":["org.apache.tika.parser.DefaultParser",
"org.apache.tika.parser.html.HtmlParser"],
"stream_content_type":["text/html"],
"date_created":"2020-11-11T21:36:38Z",
"date_current":["2020-11-17"],
"resourcename":["/mnt/Vancouver/programming/datasci/solr/test/solr_test9.html"],
"title":["APPLE BANANA Indexing Tests"],
"date_pub":"2020-11-16T21:37:18Z",
"doc_id":"bt-ic8eeW2U",
"source_url":"/mnt/Vancouver/programming/datasci/solr/test/solr_test9.html",
"dc_title":["Solr HTML Indexing Tests"],
"content_encoding":["UTF-8"],
"content_type":["application/xhtml+xml; charset=UTF-8"],
"content":[" en-us stream_size 1628 X-Parsed-By org.apache.tika.parser.DefaultParser X-Parsed-By org.apache.tika.parser.html.HtmlParser stream_content_type text/html date_created 2020-11-11T21:36:38Z resourceName /mnt/Vancouver/programming/datasci/APPLE/test/APPLE_test9.html date_pub 2020-11-16T21:37:18Z doc_id bt-ic8eeW2U source_url /mnt/Vancouver/programming/datasci/APPLE/test/APPLE_test9.html dc:title APPLE BANANA Indexing Tests Content-Encoding UTF-8 Content-Language en-us Content-Type application/xhtml+xml; charset=UTF-8 APPLE BANANA Indexing Tests Lorem ipsum dolor sit amet, consectetur adipiscing elit. "],
"div":[" div1 This text is located in div element 1. div2 This text is located in div element 2. apple This text is located in the \"apple\" (class) div element. banana This text is located in the \"banana\" (class) div element."],
"p":[" I like apples. I also like bananas. Suspendisse efficitur pulvinar elementum. My website is https://buriedtruth.com/ BuriedTruth.com . Nova Scotia is a province on the east coast of Canada. Halifax is the capital of N.S. Halifax is also N.S.'s largest city. Victoria is the capital of B.C. Vancouver is the largest city in B.C., however. Non-terminated sentence (missing period) Current date: 2020-11-17"],
"h1":[" Apples Nova Scotia British Columbia"],
"h2_t":" Bananas Capital of Nova Scotia Capital of British Columbia",
"_version_":1683814668971278336}]
}}
person
Victoria Stuart
schedule
17.11.2020