1 | /**********************************************************************
|
---|
2 | *
|
---|
3 | * querytools.cpp --
|
---|
4 | * Copyright (C) 1999 The New Zealand Digital Library Project
|
---|
5 | *
|
---|
6 | * A component of the Greenstone digital library software
|
---|
7 | * from the New Zealand Digital Library Project at the
|
---|
8 | * University of Waikato, New Zealand.
|
---|
9 | *
|
---|
10 | * This program is free software; you can redistribute it and/or modify
|
---|
11 | * it under the terms of the GNU General Public License as published by
|
---|
12 | * the Free Software Foundation; either version 2 of the License, or
|
---|
13 | * (at your option) any later version.
|
---|
14 | *
|
---|
15 | * This program is distributed in the hope that it will be useful,
|
---|
16 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
---|
17 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
---|
18 | * GNU General Public License for more details.
|
---|
19 | *
|
---|
20 | * You should have received a copy of the GNU General Public License
|
---|
21 | * along with this program; if not, write to the Free Software
|
---|
22 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
|
---|
23 | *
|
---|
24 | *********************************************************************/
|
---|
25 |
|
---|
26 | #include "querytools.h"
|
---|
27 | #include <ctype.h>
|
---|
28 | #include "unitool.h" // for is_unicode_letdig
|
---|
29 |
|
---|
30 | // sets the ct, qt, qto arguments
|
---|
31 | void set_query_type_args(ColInfoResponse_t *cinfo, cgiargsclass &args) {
|
---|
32 |
|
---|
33 | if (args["ct"].empty()) {
|
---|
34 | text_t build_type = cinfo->buildType;
|
---|
35 | if (build_type == "mgpp") {
|
---|
36 | args["ct"] = "1";
|
---|
37 | } else if (build_type == "lucene") {
|
---|
38 | args["ct"] = "2";
|
---|
39 | } else {
|
---|
40 | args["ct"] = "0";
|
---|
41 | }
|
---|
42 | }
|
---|
43 | text_t arg_ct = args["ct"];
|
---|
44 | if (arg_ct == "0") {
|
---|
45 | // mg
|
---|
46 | args["qt"] = "0";
|
---|
47 | args["qto"] = "0";
|
---|
48 | return;
|
---|
49 | }
|
---|
50 |
|
---|
51 | if (!args["qt"].empty() && !args["qto"].empty()) {
|
---|
52 | return;
|
---|
53 | }
|
---|
54 |
|
---|
55 | text_tmap::iterator check = cinfo->format.find("SearchTypes");
|
---|
56 | text_t search_types;
|
---|
57 | if(check != cinfo->format.end() && !(*check).second.empty()){
|
---|
58 | search_types = (*check).second;
|
---|
59 | } else {
|
---|
60 | // assume plain,form
|
---|
61 | if (args["qto"].empty()) args["qto"] = "3";
|
---|
62 | if (args["qt"].empty()) {
|
---|
63 | int arg_qto = args.getintarg("qto");
|
---|
64 | if (arg_qto == 2) {
|
---|
65 | args["qt"] = "1";
|
---|
66 | } else {
|
---|
67 | args["qt"] = "0";
|
---|
68 | }
|
---|
69 | }
|
---|
70 | return;
|
---|
71 | }
|
---|
72 |
|
---|
73 |
|
---|
74 | if (args["qto"].empty()) {
|
---|
75 | unsigned int type = 0;
|
---|
76 | if (findword(search_types.begin(), search_types.end(), "form") != search_types.end()) {
|
---|
77 | type |= 2;
|
---|
78 | }
|
---|
79 | if (findword(search_types.begin(), search_types.end(), "plain") != search_types.end()) {
|
---|
80 | type |= 1;
|
---|
81 | }
|
---|
82 | args.setintarg("qto", type);
|
---|
83 | }
|
---|
84 |
|
---|
85 | if (args["qt"].empty()) {
|
---|
86 | int arg_qto = args.getintarg("qto");
|
---|
87 | if (arg_qto == 2 || (arg_qto == 3 && starts_with(search_types, "form"))) {
|
---|
88 | args["qt"] = "1";
|
---|
89 | } else {
|
---|
90 | args["qt"] = "0";
|
---|
91 | }
|
---|
92 | }
|
---|
93 |
|
---|
94 |
|
---|
95 | // decide if sqlqto should be set or not
|
---|
96 | unsigned int sql_type = 0;
|
---|
97 | text_t infodb_type = cinfo->infodbType;
|
---|
98 | if ((infodb_type == "sqlite") || (infodb_type == "mssql")) {
|
---|
99 | if (findword(search_types.begin(), search_types.end(), "sqlform") != search_types.end()) {
|
---|
100 | sql_type = 1;
|
---|
101 | }
|
---|
102 | }
|
---|
103 |
|
---|
104 | if (sql_type) {
|
---|
105 | args["sqlqto"] = "1";
|
---|
106 | }
|
---|
107 | else {
|
---|
108 | args["sqlqto"] = "0";
|
---|
109 | }
|
---|
110 |
|
---|
111 |
|
---|
112 | }
|
---|
113 |
|
---|
114 | // sets the ks, ss, afs (casesupport, stemsupport, accentfoldsupport) args
|
---|
115 | void set_stem_index_args(ColInfoResponse_t *cinfo, cgiargsclass &args) {
|
---|
116 | int stemIndexes = cinfo->stemIndexes;
|
---|
117 |
|
---|
118 | if (stemIndexes & SIcasefold) {
|
---|
119 | args["ks"] = 1;
|
---|
120 | }
|
---|
121 | if (stemIndexes & SIstem) {
|
---|
122 | args["ss"] = 1;
|
---|
123 | }
|
---|
124 | if (stemIndexes & SIaccentfold) {
|
---|
125 | args["afs"] = 1;
|
---|
126 | }
|
---|
127 |
|
---|
128 | }
|
---|
129 |
|
---|
130 |
|
---|
131 |
|
---|
132 | void set_basequeryfilter_options (FilterRequest_t &request,
|
---|
133 | cgiargsclass &args)
|
---|
134 | {
|
---|
135 |
|
---|
136 | OptionValue_t option;
|
---|
137 | int arg_m = args.getintarg("m");
|
---|
138 |
|
---|
139 | option.name = "Maxdocs";
|
---|
140 | option.value = arg_m;
|
---|
141 | request.filterOptions.push_back (option);
|
---|
142 |
|
---|
143 | // option.name = "StartResults";
|
---|
144 | // option.value = args["r"];
|
---|
145 | // request.filterOptions.push_back (option);
|
---|
146 |
|
---|
147 | // option.name = "EndResults";
|
---|
148 | // int endresults = args.getintarg("o") + (args.getintarg("r") - 1);
|
---|
149 | // if ((endresults > arg_m) && (arg_m != -1)) endresults = arg_m;
|
---|
150 | // option.value = endresults;
|
---|
151 | // request.filterOptions.push_back (option);
|
---|
152 | }
|
---|
153 |
|
---|
154 |
|
---|
155 | // request.filterResultOptions and request.fields (if required) should
|
---|
156 | // be set from the calling code
|
---|
157 | void set_fulltext_queryfilter_options (FilterRequest_t &request,
|
---|
158 | const text_t &querystring,
|
---|
159 | cgiargsclass &args)
|
---|
160 | {
|
---|
161 | // better if this function, and the two-query companion function
|
---|
162 | // was implemented in queryaction.cpp
|
---|
163 | // Has to be done here to documentaction.cpp can call it directly
|
---|
164 |
|
---|
165 | request.filterName = "QueryFilter";
|
---|
166 |
|
---|
167 | OptionValue_t option;
|
---|
168 |
|
---|
169 | option.name = "Term";
|
---|
170 | option.value = querystring;
|
---|
171 | request.filterOptions.push_back (option);
|
---|
172 |
|
---|
173 | option.name = "QueryType";
|
---|
174 | option.value = (args.getintarg("t")) ? "ranked" : "boolean";
|
---|
175 | request.filterOptions.push_back (option);
|
---|
176 |
|
---|
177 | option.name = "MatchMode";
|
---|
178 | // mgpp in advanced mode, always use some query
|
---|
179 | if (args.getintarg("ct") == 1 && args.getintarg("b") == 1) {
|
---|
180 | option.value = "some";
|
---|
181 | } else {
|
---|
182 | option.value = (args.getintarg("t")) ? "some" : "all";
|
---|
183 | }
|
---|
184 | request.filterOptions.push_back (option);
|
---|
185 |
|
---|
186 | option.name = "Casefold";
|
---|
187 | option.value = (args.getintarg("k")) ? "true" : "false";
|
---|
188 | request.filterOptions.push_back (option);
|
---|
189 |
|
---|
190 | option.name = "Stem";
|
---|
191 | option.value = (args.getintarg("s")) ? "true" : "false";
|
---|
192 | request.filterOptions.push_back (option);
|
---|
193 |
|
---|
194 | option.name = "AccentFold";
|
---|
195 | option.value = (args.getintarg("af")) ? "true" : "false";
|
---|
196 | request.filterOptions.push_back (option);
|
---|
197 |
|
---|
198 | if (!args["h"].empty()) {
|
---|
199 | option.name = "Index";
|
---|
200 | option.value = args["h"];
|
---|
201 | request.filterOptions.push_back (option);
|
---|
202 | }
|
---|
203 |
|
---|
204 | if (!args["j"].empty()) {
|
---|
205 | option.name = "Subcollection";
|
---|
206 | option.value = args["j"];
|
---|
207 | request.filterOptions.push_back (option);
|
---|
208 | }
|
---|
209 |
|
---|
210 | if (!args["n"].empty()) {
|
---|
211 | option.name = "Language";
|
---|
212 | option.value = args["n"];
|
---|
213 | request.filterOptions.push_back (option);
|
---|
214 | }
|
---|
215 |
|
---|
216 | if (!args["g"].empty()) { // granularity for mgpp
|
---|
217 | option.name = "Level";
|
---|
218 | option.value = args["g"];
|
---|
219 | request.filterOptions.push_back (option);
|
---|
220 | }
|
---|
221 |
|
---|
222 | if (!args["fs"].empty()) { // filter string for lucene
|
---|
223 | option.name = "FilterString";
|
---|
224 | option.value = args["fs"];
|
---|
225 | request.filterOptions.push_back (option);
|
---|
226 | }
|
---|
227 |
|
---|
228 | if (!args["sf"].empty()) { // sort field for lucene
|
---|
229 | option.name = "SortField";
|
---|
230 | option.value = args["sf"];
|
---|
231 | request.filterOptions.push_back (option);
|
---|
232 | }
|
---|
233 |
|
---|
234 | if (!args["fuzziness"].empty() && args["fuzziness"] != "100") { // fuzziness value for lucene
|
---|
235 | option.name = "Fuzziness";
|
---|
236 | option.value = (text_t) "0." + args["fuzziness"];
|
---|
237 | request.filterOptions.push_back (option);
|
---|
238 | }
|
---|
239 |
|
---|
240 | set_basequeryfilter_options(request, args);
|
---|
241 | }
|
---|
242 |
|
---|
243 |
|
---|
244 |
|
---|
245 | void set_fulltext_queryfilter_options (FilterRequest_t &request,
|
---|
246 | const text_t &querystring1,
|
---|
247 | const text_t &querystring2,
|
---|
248 | cgiargsclass &args)
|
---|
249 | {
|
---|
250 |
|
---|
251 | set_fulltext_queryfilter_options (request, querystring1, args);
|
---|
252 |
|
---|
253 | // fill in the second query if needed
|
---|
254 | if (!args["cq2"].empty()) {
|
---|
255 | OptionValue_t option;
|
---|
256 |
|
---|
257 | option.name = "CombineQuery";
|
---|
258 | option.value = args["cq2"];
|
---|
259 | request.filterOptions.push_back (option);
|
---|
260 |
|
---|
261 | option.name = "Term";
|
---|
262 | option.value = querystring2;
|
---|
263 | request.filterOptions.push_back (option);
|
---|
264 |
|
---|
265 | option.name = "QueryType";
|
---|
266 | option.value = (args.getintarg("t")) ? "ranked" : "boolean";
|
---|
267 | request.filterOptions.push_back (option);
|
---|
268 |
|
---|
269 | option.name = "Casefold";
|
---|
270 | option.value = (args.getintarg("k")) ? "true" : "false";
|
---|
271 | request.filterOptions.push_back (option);
|
---|
272 |
|
---|
273 | option.name = "Stem";
|
---|
274 | option.value = (args.getintarg("s")) ? "true" : "false";
|
---|
275 | request.filterOptions.push_back (option);
|
---|
276 |
|
---|
277 | option.name = "AccentFold";
|
---|
278 | option.value = (args.getintarg("af")) ? "true" : "false";
|
---|
279 | request.filterOptions.push_back (option);
|
---|
280 |
|
---|
281 | if (!args["h2"].empty()) {
|
---|
282 | option.name = "Index";
|
---|
283 | option.value = args["h2"];
|
---|
284 | request.filterOptions.push_back (option);
|
---|
285 | }
|
---|
286 |
|
---|
287 | if (!args["j2"].empty()) {
|
---|
288 | option.name = "Subcollection";
|
---|
289 | option.value = args["j2"];
|
---|
290 | request.filterOptions.push_back (option);
|
---|
291 | }
|
---|
292 |
|
---|
293 | if (!args["n2"].empty()) {
|
---|
294 | option.name = "Language";
|
---|
295 | option.value = args["n2"];
|
---|
296 | request.filterOptions.push_back (option);
|
---|
297 | }
|
---|
298 | }
|
---|
299 |
|
---|
300 | // this is probably redundant, as first line to this method will have
|
---|
301 | // already caused it to invoke set_basequeryfilter_options
|
---|
302 |
|
---|
303 | set_basequeryfilter_options(request, args);
|
---|
304 | }
|
---|
305 |
|
---|
306 |
|
---|
307 |
|
---|
308 | // request.filterResultOptions and request.fields (if required) should
|
---|
309 | // be set from the calling code
|
---|
310 | void set_sql_queryfilter_options (FilterRequest_t &request,
|
---|
311 | cgiargsclass &args)
|
---|
312 | {
|
---|
313 | if (!args["sqlsf"].empty()) { // sort field for lucene
|
---|
314 | OptionValue_t option;
|
---|
315 |
|
---|
316 | option.name = "SortField";
|
---|
317 | option.value = args["sqlsf"];
|
---|
318 | request.filterOptions.push_back (option);
|
---|
319 | }
|
---|
320 |
|
---|
321 | set_basequeryfilter_options(request, args);
|
---|
322 | }
|
---|
323 |
|
---|
324 |
|
---|
325 | bool is_special_character(int indexer_type, unsigned short character) {
|
---|
326 | // mgpp
|
---|
327 | if (indexer_type == 1) {
|
---|
328 | return (character == '#' || character == '/' || character == '*');
|
---|
329 | }
|
---|
330 | // lucene
|
---|
331 | else if (indexer_type == 2) {
|
---|
332 | return (character == '?' || character == '*' || character == '~' ||
|
---|
333 | character == '^');
|
---|
334 | }
|
---|
335 | return false;
|
---|
336 | }
|
---|
337 |
|
---|
338 | // This function removes boolean operators from simple searches, and segments
|
---|
339 | // chinese characters if segment=true
|
---|
340 | void format_querystring (text_t &querystring, int querymode, bool segment) {
|
---|
341 | text_t formattedstring;
|
---|
342 |
|
---|
343 | // advanced search, no segmenting, don't need to do anything
|
---|
344 | if (querymode == 1 && !segment) return;
|
---|
345 |
|
---|
346 | text_t::const_iterator here = querystring.begin();
|
---|
347 | text_t::const_iterator end = querystring.end();
|
---|
348 |
|
---|
349 | // space is used to insert spaces between Chinese
|
---|
350 | // characters. No space is needed before the first
|
---|
351 | // Chinese character.
|
---|
352 | bool space = false;
|
---|
353 |
|
---|
354 | // want to remove ()|!& from querystring so boolean queries are just
|
---|
355 | // "all the words" queries (unless querymode is advanced)
|
---|
356 | while (here != end) {
|
---|
357 | if ((querymode == 0) && (*here == '(' || *here == ')' || *here == '|' ||
|
---|
358 | *here == '!' || *here == '&')) {
|
---|
359 | formattedstring.push_back(' ');
|
---|
360 | } else if (segment) {
|
---|
361 | if ((*here >= 0x2e80 && *here <= 0xd7a3) ||
|
---|
362 | ( *here >= 0xf900 && *here <= 0xfa6a)) {
|
---|
363 | /* text_t not big enough to handle these. */
|
---|
364 | /* (*here >= 0x20000 && *here <= 0x2a6d6) ||
|
---|
365 | (*here >= 0x2f800 && *here <= 0x2fa1d)) { */
|
---|
366 |
|
---|
367 | // CJK character
|
---|
368 | if (!space) formattedstring.push_back (0x200b); // zero width space
|
---|
369 | formattedstring.push_back (*here);
|
---|
370 | formattedstring.push_back (0x200b);
|
---|
371 | space = true;
|
---|
372 | } else {
|
---|
373 |
|
---|
374 | // non-Chinese character
|
---|
375 | formattedstring.push_back (*here);
|
---|
376 | space = false;
|
---|
377 |
|
---|
378 | }
|
---|
379 |
|
---|
380 | } else {
|
---|
381 | formattedstring.push_back (*here);
|
---|
382 | }
|
---|
383 | ++here;
|
---|
384 | }
|
---|
385 | querystring = formattedstring;
|
---|
386 | }
|
---|
387 |
|
---|
388 | // turn query string into terms separated by spaces.
|
---|
389 | // still working on this...
|
---|
390 | text_t get_plain_query_terms(const text_t &querystring, const text_t &arg_ct) {
|
---|
391 | text_t::const_iterator here = querystring.begin();
|
---|
392 | text_t::const_iterator end = querystring.end();
|
---|
393 |
|
---|
394 | // lets look for [] and () first - these are a pain.
|
---|
395 | text_t::const_iterator bracket;
|
---|
396 | text_t query_no_brackets = "";
|
---|
397 |
|
---|
398 | // mgpp brackets: [xxx]:TI
|
---|
399 | if (findchar(here, end, '[') != end) {
|
---|
400 | while ((bracket = findchar(here, end, '[')) != end) {
|
---|
401 | // get the first bit
|
---|
402 | query_no_brackets += substr(here, bracket);
|
---|
403 | bracket++;
|
---|
404 | here = bracket;
|
---|
405 | // get the end bracket
|
---|
406 | bracket = findchar(here, end, ']');
|
---|
407 | query_no_brackets += substr(here, bracket);
|
---|
408 | // skip the :TI bits
|
---|
409 | while (bracket != end // do bracket != end test first, ELSE when bracket = end, we're past the string, in
|
---|
410 | && *bracket != ' ') { // which case *bracket becomes an invalid operation that causes the server to crash
|
---|
411 | bracket++;
|
---|
412 | }
|
---|
413 | here = bracket;
|
---|
414 | }
|
---|
415 | if (here != end) {
|
---|
416 | query_no_brackets += substr(here,end);
|
---|
417 | }
|
---|
418 | } else if (findchar(here, end, '(') != end) {
|
---|
419 | // lucene brackets TI:(xxx)
|
---|
420 | while ((bracket = findchar(here, end, '(')) != end) {
|
---|
421 | // back up the field name
|
---|
422 | text_t::const_iterator old_bracket = bracket;
|
---|
423 | while (bracket != here && *bracket != ' ') { // order of tests in condition matters (see long comment above)
|
---|
424 | --bracket;
|
---|
425 | }
|
---|
426 | if (bracket != here) {
|
---|
427 | // get the first bit
|
---|
428 | query_no_brackets += substr(here, bracket+1);
|
---|
429 | }
|
---|
430 | here = old_bracket +1;
|
---|
431 | // get the end bracket
|
---|
432 | bracket = findchar(here, end, ')');
|
---|
433 | query_no_brackets += substr(here, bracket);
|
---|
434 | if (bracket != end) {
|
---|
435 | here = bracket+1;
|
---|
436 | }
|
---|
437 | }
|
---|
438 | if (here != end) {
|
---|
439 | query_no_brackets += substr(here,end);
|
---|
440 | }
|
---|
441 | } else {
|
---|
442 | // was no brackets
|
---|
443 | query_no_brackets = querystring;
|
---|
444 | }
|
---|
445 |
|
---|
446 |
|
---|
447 | if (arg_ct == "2") { // lucene
|
---|
448 | // look for AND OR NOT and remove
|
---|
449 | here = query_no_brackets.begin();
|
---|
450 | end = query_no_brackets.end();
|
---|
451 | text_tlist terms;
|
---|
452 | splitword(here, end, "AND", terms);
|
---|
453 | joinchar(terms, ' ', query_no_brackets);
|
---|
454 | here = query_no_brackets.begin();
|
---|
455 | end = query_no_brackets.end();
|
---|
456 | splitword(here, end, "OR", terms);
|
---|
457 | joinchar(terms, ' ', query_no_brackets);
|
---|
458 | here = query_no_brackets.begin();
|
---|
459 | end = query_no_brackets.end();
|
---|
460 | splitword(here, end, "NOT", terms);
|
---|
461 | joinchar(terms, ' ', query_no_brackets);
|
---|
462 |
|
---|
463 | }
|
---|
464 | text_t terms = "";
|
---|
465 | bool space = false;
|
---|
466 | here = query_no_brackets.begin();
|
---|
467 | end = query_no_brackets.end();
|
---|
468 |
|
---|
469 | while (here != end) {
|
---|
470 | if (*here == '#' || *here == '/') {
|
---|
471 | // skip over #is /10 etc
|
---|
472 | ++here;
|
---|
473 | while (here != end && *here != ' ') {
|
---|
474 | ++here;
|
---|
475 | }
|
---|
476 | if (here == end) break;
|
---|
477 | }
|
---|
478 | if (is_unicode_letdig(*here)) {
|
---|
479 | terms.push_back(*here);
|
---|
480 | space = false;
|
---|
481 | } else {
|
---|
482 | if (!space) {
|
---|
483 | terms.push_back(' ');
|
---|
484 | space = true;
|
---|
485 | }
|
---|
486 | }
|
---|
487 | ++here;
|
---|
488 | }
|
---|
489 | return trim(terms);
|
---|
490 |
|
---|
491 | }
|
---|
492 |
|
---|
493 | // search history tool
|
---|
494 | // also used for form query macros
|
---|
495 | text_t escape_quotes(const text_t &querystring) {
|
---|
496 |
|
---|
497 | text_t::const_iterator here = querystring.begin();
|
---|
498 | text_t::const_iterator end = querystring.end();
|
---|
499 |
|
---|
500 | text_t escquery = "";
|
---|
501 | while (here != end) {
|
---|
502 | if (*here != '\'' && *here != '\"' && *here != '\n' && *here != '\r') escquery.push_back(*here);
|
---|
503 | else if (*here == '\n' || *here == '\r') {
|
---|
504 | escquery.push_back(' ');
|
---|
505 | } else {
|
---|
506 | escquery +="\\\\";
|
---|
507 | escquery.push_back(*here);
|
---|
508 | }
|
---|
509 |
|
---|
510 | ++here;
|
---|
511 | }
|
---|
512 | return escquery;
|
---|
513 |
|
---|
514 | }
|
---|
515 |
|
---|
516 | // Parses the terms into words, and adds #si if necessary
|
---|
517 | text_t addstemcase(const text_t &terms, const text_t &stem, const text_t &fold,
|
---|
518 | const int indexer_type) {
|
---|
519 |
|
---|
520 | // the default stem and case are set to 0 if this is being used, so we are only adding on qualifiers if stem or fold is 1.
|
---|
521 | if (stem == "0" && fold == "0") {
|
---|
522 | return terms;
|
---|
523 | }
|
---|
524 | // this is only for mgpp collections, shouldn't be called for anything else
|
---|
525 | if (indexer_type != 1) {
|
---|
526 | return terms;
|
---|
527 | }
|
---|
528 |
|
---|
529 | text_t outtext;
|
---|
530 | text_t word;
|
---|
531 |
|
---|
532 | text_t::const_iterator here = terms.begin();
|
---|
533 | text_t::const_iterator end = terms.end();
|
---|
534 |
|
---|
535 | while (here !=end) {
|
---|
536 |
|
---|
537 | if (is_unicode_letdig(*here) || is_special_character(indexer_type, *here)) {
|
---|
538 | // not word boundary
|
---|
539 | word.push_back(*here);
|
---|
540 | ++here;
|
---|
541 | }
|
---|
542 | else {
|
---|
543 | // found word boundary
|
---|
544 | if (!word.empty() ) {
|
---|
545 | if (starts_with(word, "NEAR") || starts_with(word, "WITHIN")) {
|
---|
546 | outtext += word;
|
---|
547 | word.clear();
|
---|
548 | }
|
---|
549 | else {
|
---|
550 | word += "#";
|
---|
551 | if (stem == "1") word += "s";
|
---|
552 | if (fold == "1") word += "i";
|
---|
553 | outtext += word;
|
---|
554 | word.clear();
|
---|
555 | }
|
---|
556 | }
|
---|
557 | // this only used in advanced form, so we leave in boolean operators
|
---|
558 | if (*here == '\"' || *here == '&' || *here == '|' || *here == '!' ||
|
---|
559 | *here == '(' || *here == ')' || is_unicode_space(*here)) {
|
---|
560 | outtext.push_back(*here);
|
---|
561 | }
|
---|
562 | ++here;
|
---|
563 | }
|
---|
564 | }
|
---|
565 |
|
---|
566 | // get last word
|
---|
567 | if (!word.empty()) {
|
---|
568 | word += "#";
|
---|
569 | if (stem == "1") word += "s";
|
---|
570 | if (fold == "1") word += "i";
|
---|
571 | word += " ";
|
---|
572 | outtext += word;
|
---|
573 | }
|
---|
574 | return outtext;
|
---|
575 | }
|
---|
576 |
|
---|
577 |
|
---|
578 | // some query form parsing functions for use with mgpp & lucene
|
---|
579 |
|
---|
580 | void parse_reg_query_form(text_t &querystring, cgiargsclass &args, bool segment)
|
---|
581 | {
|
---|
582 | querystring.clear();
|
---|
583 |
|
---|
584 | int argct = args.getintarg("ct");
|
---|
585 | int argt = args.getintarg("t");// t=0 -and, t=1 - or
|
---|
586 | int argb = args.getintarg("b");
|
---|
587 |
|
---|
588 | text_t combine;
|
---|
589 |
|
---|
590 | // lucene uses global combine, so only need this for mgpp
|
---|
591 | if (argct==1) {
|
---|
592 | if (argt == 0) combine = "&";
|
---|
593 | else combine = "|";
|
---|
594 | }
|
---|
595 |
|
---|
596 | text_t field = args["fqf"];
|
---|
597 | if (field.empty()) return; // no query
|
---|
598 | text_tarray fields;
|
---|
599 | splitchar(field.begin(), field.end(), ',', fields);
|
---|
600 |
|
---|
601 | text_t value = args["fqv"];
|
---|
602 | if (value.empty()) return; // somethings wrong
|
---|
603 | text_tarray values;
|
---|
604 | splitchar(value.begin(), value.end(), ',', values);
|
---|
605 |
|
---|
606 |
|
---|
607 | for (int i=0; i< values.size(); ++i) {
|
---|
608 | if (!values[i].empty()) {
|
---|
609 | text_t this_value = values[i];
|
---|
610 |
|
---|
611 | // remove operators for simple search, segments text if necessary
|
---|
612 | format_querystring(this_value, argb, segment);
|
---|
613 |
|
---|
614 | // add tag info for this field (and other processing)
|
---|
615 | format_field_info(this_value, fields[i], argct, argt, argb);
|
---|
616 |
|
---|
617 | // add into query string
|
---|
618 | if (argct == 2) {
|
---|
619 | // lucene
|
---|
620 | // we don't worry about AND/OR, cos this is done by defaultcombineoperator
|
---|
621 | querystring += this_value+" ";
|
---|
622 | } else {
|
---|
623 | // mgpp
|
---|
624 | if (!querystring.empty()) {
|
---|
625 | querystring += " "+ combine+ " ";
|
---|
626 | }
|
---|
627 | querystring += this_value;
|
---|
628 | }
|
---|
629 | }
|
---|
630 | }
|
---|
631 | }
|
---|
632 |
|
---|
633 |
|
---|
634 | void parse_adv_query_form(text_t &querystring, cgiargsclass &args, bool segment){
|
---|
635 | querystring.clear();
|
---|
636 |
|
---|
637 | const int argct = args.getintarg("ct");
|
---|
638 | int argt = 0;// arg t is either not used (lucene) or used for natural/ranked (mgpp), so we set it to 0 = AND, by default
|
---|
639 | int argb = args.getintarg("b");
|
---|
640 | text_t combine;
|
---|
641 | if (argct==1) {
|
---|
642 | combine = "&";
|
---|
643 | }
|
---|
644 | else { // lucene
|
---|
645 | combine = "AND";
|
---|
646 | }
|
---|
647 |
|
---|
648 | text_t field = args["fqf"];
|
---|
649 | if (field.empty()) return; // no query
|
---|
650 | text_tarray fields;
|
---|
651 | splitchar(field.begin(), field.end(), ',', fields);
|
---|
652 |
|
---|
653 | text_t value = args["fqv"];
|
---|
654 | if (value.empty()) return; // somethings wrong
|
---|
655 | text_tarray values;
|
---|
656 | splitchar(value.begin(), value.end(), ',', values);
|
---|
657 |
|
---|
658 | text_t comb = args["fqc"];
|
---|
659 | if (comb.empty()) return; //somethings wrong
|
---|
660 | text_tarray combs;
|
---|
661 | splitchar(comb.begin(), comb.end(), ',', combs);
|
---|
662 |
|
---|
663 | text_tarray stems;
|
---|
664 | text_tarray folds;
|
---|
665 | if (argct == 1) {// mgpp - lucene doesn't do stem/case
|
---|
666 | text_t stem = args["fqs"];
|
---|
667 | if (stem.empty()) return; // somethings wrong
|
---|
668 | splitchar(stem.begin(), stem.end(), ',', stems);
|
---|
669 |
|
---|
670 | text_t fold = args["fqk"];
|
---|
671 | if (fold.empty()) return; // somethings wrong
|
---|
672 | splitchar(fold.begin(), fold.end(), ',', folds);
|
---|
673 | }
|
---|
674 |
|
---|
675 | for(int i=0; i< values.size(); ++i) {
|
---|
676 | if (!values[i].empty()) {
|
---|
677 | if (i!=0) {
|
---|
678 | if (argct==1) {
|
---|
679 | if (combs[i-1]=="and") combine = "&";
|
---|
680 | else if (combs[i-1]=="or")combine = "|";
|
---|
681 | else if (combs[i-1]=="not")combine = "!";
|
---|
682 | }
|
---|
683 | else { // lucene
|
---|
684 | if (combs[i-1]=="and") combine = "AND";
|
---|
685 | else if (combs[i-1]=="or")combine = "OR";
|
---|
686 | else if (combs[i-1]=="not")combine = "NOT";
|
---|
687 | }
|
---|
688 | }
|
---|
689 | text_t this_value = values[i];
|
---|
690 | // remove operators for simple search, segments text if necessary
|
---|
691 | format_querystring(this_value, argb, segment);
|
---|
692 | if (argct == 1) { // mgpp only
|
---|
693 | this_value = addstemcase(this_value, stems[i], folds[i], argct);
|
---|
694 | }
|
---|
695 | // add tag info for this field (and other processing)
|
---|
696 | format_field_info(this_value, fields[i], argct, argt, argb);
|
---|
697 | // add into query string
|
---|
698 | if (!querystring.empty()) {
|
---|
699 | querystring += " "+ combine+ " ";
|
---|
700 | }
|
---|
701 | querystring += this_value;
|
---|
702 |
|
---|
703 | }
|
---|
704 | }
|
---|
705 | }
|
---|
706 |
|
---|
707 |
|
---|
708 | // SQL versions for parsing query form
|
---|
709 |
|
---|
710 | void parse_sqlreg_query_form(text_t &querystring, cgiargsclass &args, bool segment)
|
---|
711 | {
|
---|
712 | querystring.clear();
|
---|
713 |
|
---|
714 | int argt = args.getintarg("t");// t=0 -and, t=1 - or
|
---|
715 | int argb = args.getintarg("b");
|
---|
716 |
|
---|
717 | text_t combine;
|
---|
718 |
|
---|
719 | if (argt == 0) combine = "AND";
|
---|
720 | else combine = "OR";
|
---|
721 |
|
---|
722 | text_t field = args["sqlfqf"];
|
---|
723 | if (field.empty()) return; // no query
|
---|
724 | text_tarray fields;
|
---|
725 | splitchar(field.begin(), field.end(), ',', fields);
|
---|
726 |
|
---|
727 | text_t sqlcomb = args["sqlfqc"];
|
---|
728 | if (sqlcomb.empty()) return; //somethings wrong
|
---|
729 | text_tarray sqlcombs;
|
---|
730 | splitchar(sqlcomb.begin(), sqlcomb.end(), ',', sqlcombs);
|
---|
731 |
|
---|
732 | text_t value = args["fqv"];
|
---|
733 | if (value.empty()) return; // somethings wrong
|
---|
734 | text_tarray values;
|
---|
735 | splitchar(value.begin(), value.end(), ',', values);
|
---|
736 |
|
---|
737 |
|
---|
738 | for (int i=0; i< values.size(); ++i) {
|
---|
739 | if (!values[i].empty()) {
|
---|
740 | text_t this_value;
|
---|
741 | const text_t STARTINGWITH_CONDITION = "STARTINGWITH";
|
---|
742 | const text_t LIKE_CONDITION = "LIKE";
|
---|
743 |
|
---|
744 | //Change the STARTINGWITH operator to 'LIKE' and then adds '%' to the end of the value field
|
---|
745 | //in order to search a field starting with certain words.
|
---|
746 | if (sqlcombs[i] == STARTINGWITH_CONDITION)
|
---|
747 | {this_value = values[i];
|
---|
748 | this_value += "%";
|
---|
749 | // remove operators for simple search, segments text if necessary
|
---|
750 | format_querystring(this_value, argb, segment);
|
---|
751 | // add tag info for this field (and other processing)
|
---|
752 | format_field_info_sql(this_value, fields[i], LIKE_CONDITION, argt, argb);}
|
---|
753 |
|
---|
754 | else
|
---|
755 | {this_value = values[i];
|
---|
756 | // remove operators for simple search, segments text if necessary
|
---|
757 | format_querystring(this_value, argb, segment);
|
---|
758 | // add tag info for this field (and other processing)
|
---|
759 | format_field_info_sql(this_value, fields[i], sqlcombs[i], argt, argb);}
|
---|
760 |
|
---|
761 |
|
---|
762 | const text_t DISTINCT_SELECT_WHERE = "SELECT DISTINCT docOID FROM document_metadata WHERE ";
|
---|
763 |
|
---|
764 | if (querystring.empty()) {
|
---|
765 | // first query term
|
---|
766 | querystring = DISTINCT_SELECT_WHERE + this_value;
|
---|
767 | }
|
---|
768 | else {
|
---|
769 | this_value = DISTINCT_SELECT_WHERE + this_value;
|
---|
770 |
|
---|
771 | if (combine=="AND") {
|
---|
772 | // INNER JOIN to restrict to only matching docOIDs
|
---|
773 | querystring = "SELECT docOID FROM (" + querystring + ")"
|
---|
774 | + " INNER JOIN (" + this_value +") USING (docOID)";
|
---|
775 | }
|
---|
776 | else if (combine=="OR") {
|
---|
777 | // Union to allow union of the two
|
---|
778 | querystring = querystring + " UNION " + this_value;
|
---|
779 | }
|
---|
780 | }
|
---|
781 | }
|
---|
782 | }
|
---|
783 | }
|
---|
784 |
|
---|
785 |
|
---|
786 | void parse_sqladv_query_form(text_t &querystring, cgiargsclass &args,
|
---|
787 | bool segment)
|
---|
788 | {
|
---|
789 | querystring.clear();
|
---|
790 |
|
---|
791 | int argt = 0; // set it to 0 = AND, by default
|
---|
792 | int argb = args.getintarg("b");
|
---|
793 | text_t combine = "AND";
|
---|
794 |
|
---|
795 | text_t field = args["sqlfqf"];
|
---|
796 |
|
---|
797 | if (field.empty()) return; // no query
|
---|
798 | text_tarray fields;
|
---|
799 | splitchar(field.begin(), field.end(), ',', fields);
|
---|
800 |
|
---|
801 | text_t sqlcomb = args["sqlfqc"];
|
---|
802 | if (sqlcomb.empty()) return; //somethings wrong
|
---|
803 | text_tarray sqlcombs;
|
---|
804 | splitchar(sqlcomb.begin(), sqlcomb.end(), ',', sqlcombs);
|
---|
805 |
|
---|
806 | text_t value = args["fqv"];
|
---|
807 | if (value.empty()) return; // somethings wrong
|
---|
808 | text_tarray values;
|
---|
809 | splitchar(value.begin(), value.end(), ',', values);
|
---|
810 |
|
---|
811 | text_t comb = args["fqc"];
|
---|
812 | if (comb.empty()) return; //somethings wrong
|
---|
813 | text_tarray combs;
|
---|
814 | splitchar(comb.begin(), comb.end(), ',', combs);
|
---|
815 |
|
---|
816 | for(int i=0; i< values.size(); ++i) {
|
---|
817 | if (!values[i].empty()) {
|
---|
818 | if (i>0) {
|
---|
819 | if (combs[i-1]=="and") { combine = "AND"; }
|
---|
820 | else if (combs[i-1]=="or") { combine = "OR"; }
|
---|
821 | else if (combs[i-1]=="not") { combine = "NOT"; }
|
---|
822 | }
|
---|
823 | text_t this_value;
|
---|
824 | const text_t STARTINGWITH_CONDITION = "STARTINGWITH";
|
---|
825 | const text_t LIKE_CONDITION = "LIKE";
|
---|
826 |
|
---|
827 | //Change the STARTINGWITH operator to 'LIKE' and then adds '%' to the end of the value field
|
---|
828 | //in order to search a field starting with certain words.
|
---|
829 | if (sqlcombs[i] == STARTINGWITH_CONDITION)
|
---|
830 | {this_value = values[i];
|
---|
831 | this_value += "%";
|
---|
832 | // remove operators for simple search, segments text if necessary
|
---|
833 | format_querystring(this_value, argb, segment);
|
---|
834 | // add tag info for this field (and other processing)
|
---|
835 | format_field_info_sql(this_value, fields[i], LIKE_CONDITION, argt, argb);}
|
---|
836 |
|
---|
837 | else
|
---|
838 | {this_value = values[i];
|
---|
839 | // remove operators for simple search, segments text if necessary
|
---|
840 | format_querystring(this_value, argb, segment);
|
---|
841 | // add tag info for this field (and other processing)
|
---|
842 | format_field_info_sql(this_value, fields[i], sqlcombs[i], argt, argb);}
|
---|
843 |
|
---|
844 | const text_t DISTINCT_SELECT_WHERE = "SELECT DISTINCT docOID FROM document_metadata WHERE ";
|
---|
845 |
|
---|
846 | if (querystring.empty()) {
|
---|
847 | // first query term
|
---|
848 | querystring = DISTINCT_SELECT_WHERE + this_value;
|
---|
849 | }
|
---|
850 | else {
|
---|
851 | this_value = DISTINCT_SELECT_WHERE + this_value;
|
---|
852 |
|
---|
853 | if (combine=="AND") {
|
---|
854 | // INNER JOIN to restrict to only matching docOIDs
|
---|
855 | querystring = "SELECT docOID FROM (" + querystring + ")"
|
---|
856 | + " INNER JOIN (" + this_value +") USING (docOID)";
|
---|
857 | }
|
---|
858 | else if (combine=="OR") {
|
---|
859 | // Union to allow union of the two
|
---|
860 | querystring = querystring + " UNION " + this_value;
|
---|
861 | }
|
---|
862 | else {
|
---|
863 | cerr << "Unsupported combination operation: " << combine << endl;
|
---|
864 | }
|
---|
865 | }
|
---|
866 |
|
---|
867 | }
|
---|
868 | }
|
---|
869 | }
|
---|
870 |
|
---|
871 |
|
---|
872 |
|
---|
873 |
|
---|
874 | // Extended addqueryelem for Human Info project
|
---|
875 | void addqueryelem_ex(text_t &querystring, const text_t &tag,
|
---|
876 | const text_t &terms, const text_t &stem,
|
---|
877 | const text_t &fold,
|
---|
878 | const text_t& combine, const text_t& word_combine) {
|
---|
879 |
|
---|
880 | if (!querystring.empty()) { // have to put and/or
|
---|
881 | querystring += " " + combine + " ";
|
---|
882 | }
|
---|
883 | text_t outtext; outtext.reserve(512);
|
---|
884 | text_t word; word.reserve(100);
|
---|
885 | //unsigned short c;
|
---|
886 | text_t::const_iterator here = terms.begin();
|
---|
887 | text_t::const_iterator end = terms.end();
|
---|
888 | bool inquote = false, firstword = true;
|
---|
889 |
|
---|
890 | text_t word2; word2.reserve(256);
|
---|
891 |
|
---|
892 | while (here !=end) {
|
---|
893 | if (is_unicode_space(*here)) {
|
---|
894 | if (word2 == "AND") { word2.clear(); word2.push_back(7527); word2.appendcarr("AND", 3); word2.push_back(7527); }
|
---|
895 | else if (word2 == "OR") { word2.clear(); word2.push_back(7527); word2.appendcarr("OR", 2); word2.push_back(7527); }
|
---|
896 | else if (word2 == "NOT") { word2.clear(); word2.push_back(7527); word2.appendcarr("NOT", 3); word2.push_back(7527); }
|
---|
897 | else if (word2 == "NEAR") { word2.clear(); word2.push_back(7527); word2.appendcarr("NEAR", 4); word2.push_back(7527); }
|
---|
898 | else if (word2 == "WITHIN") { word2.clear(); word2.push_back(7527); word2.appendcarr("WITHIN", 6); word2.push_back(7527); }
|
---|
899 | if (inquote) {
|
---|
900 | word2.push_back(*here);
|
---|
901 | }
|
---|
902 | word.append(word2); word2.clear();
|
---|
903 |
|
---|
904 | if (!inquote && !word.empty() ) {
|
---|
905 | // found word boundary
|
---|
906 |
|
---|
907 | if (stem == "1" || fold =="1") {
|
---|
908 | word += "#";
|
---|
909 | if (stem == "1") word += "s";
|
---|
910 | //else word += "u";
|
---|
911 |
|
---|
912 | if (fold == "1") word += "i";
|
---|
913 | //else word += "c";
|
---|
914 | }
|
---|
915 | if (firstword) {
|
---|
916 | firstword = false;
|
---|
917 | } else {
|
---|
918 | outtext += " " + word_combine + " ";
|
---|
919 | }
|
---|
920 | outtext += "[" + word + "]:"+tag;
|
---|
921 | word.clear();
|
---|
922 | }
|
---|
923 | ++here;
|
---|
924 | } else if (*here == '\"') {
|
---|
925 | word2.push_back(*here);
|
---|
926 | inquote = !inquote;
|
---|
927 | ++here;
|
---|
928 | } else {
|
---|
929 | // not word boundary
|
---|
930 | word2.push_back(*here);
|
---|
931 | ++here;
|
---|
932 | }
|
---|
933 | }
|
---|
934 |
|
---|
935 | // get last word
|
---|
936 | if (!word2.empty()) {
|
---|
937 | if (word2 == "AND") { word2.clear(); word2.push_back(7527); word2.appendcarr("AND", 3); word2.push_back(7527); }
|
---|
938 | else if (word2 == "OR") { word2.clear(); word2.push_back(7527); word2.appendcarr("OR", 2); word2.push_back(7527); }
|
---|
939 | else if (word2 == "NOT") { word2.clear(); word2.push_back(7527); word2.appendcarr("NOT", 3); word2.push_back(7527); }
|
---|
940 | else if (word2 == "NEAR") { word2.clear(); word2.push_back(7527); word2.appendcarr("NEAR", 4); word2.push_back(7527); }
|
---|
941 | else if (word2 == "WITHIN") { word2.clear(); word2.push_back(7527); word2.appendcarr("WITHIN", 6); word2.push_back(7527); }
|
---|
942 | word.append(word2); word2.clear();
|
---|
943 |
|
---|
944 | if (stem == "1"|| fold == "1") {
|
---|
945 | word += "#";
|
---|
946 | if (stem == "1") word += "s";
|
---|
947 | //else word += "u";
|
---|
948 |
|
---|
949 | if (fold == "1") word += "i";
|
---|
950 | //else word += "c";
|
---|
951 | }
|
---|
952 | if (!outtext.empty()) outtext += " " + word_combine + " ";
|
---|
953 | outtext += "[" + word + "]:"+tag;
|
---|
954 | }
|
---|
955 | querystring += "(" + outtext + ")";
|
---|
956 | }
|
---|
957 |
|
---|
958 | void add_field_info(text_t &querystring, const text_t &tag, int type) {
|
---|
959 |
|
---|
960 | if (tag == "") return; // do nothing
|
---|
961 | if (tag == "ZZ" && type == 1) return; // mgpp doesn't use ZZ tag internally
|
---|
962 | if (type == 1) { //mgpp
|
---|
963 | querystring = "["+querystring+"]:"+tag;
|
---|
964 | } else if (type == 2) { // lucene
|
---|
965 | querystring = tag+":("+querystring+")";
|
---|
966 | }
|
---|
967 |
|
---|
968 | }
|
---|
969 |
|
---|
970 |
|
---|
971 | void add_field_info_sql(text_t &querystring, const text_t &tagseq,
|
---|
972 | const text_t& sqlcomb)
|
---|
973 | {
|
---|
974 |
|
---|
975 | if (tagseq == "") return; // do nothing
|
---|
976 |
|
---|
977 | text_t element_in = "(element IN (";
|
---|
978 |
|
---|
979 | text_tlist mdterms;
|
---|
980 |
|
---|
981 | splitword(tagseq.begin(), tagseq.end(), "/", mdterms);
|
---|
982 |
|
---|
983 | text_t tags_in = "";
|
---|
984 |
|
---|
985 | while (!mdterms.empty()) {
|
---|
986 | text_t tag = mdterms.front();
|
---|
987 | mdterms.pop_front();
|
---|
988 |
|
---|
989 | if (!tag.empty()) {
|
---|
990 |
|
---|
991 | // remove "ex." prefix, but only if there are no other metadata set qualifiers
|
---|
992 | // in the metaname, since we want to retain prefixes like "ex.dc." as-is
|
---|
993 | text_t::iterator period = findchar(tag.begin(), tag.end(), '.');
|
---|
994 | text_t::iterator lastperiod = findlastchar(tag.begin(), tag.end(), '.');
|
---|
995 |
|
---|
996 | if (tag.size()>3 && (substr(tag.begin(), tag.begin()+3) == "ex.") && period == lastperiod) {
|
---|
997 | tag = substr (tag.begin()+3, tag.end());
|
---|
998 | }
|
---|
999 |
|
---|
1000 | if (!tags_in.empty()) {
|
---|
1001 | tags_in += ",";
|
---|
1002 | }
|
---|
1003 |
|
---|
1004 | tags_in += "'" + tag + "'";
|
---|
1005 | }
|
---|
1006 | }
|
---|
1007 |
|
---|
1008 | element_in += tags_in + ") AND (";
|
---|
1009 |
|
---|
1010 |
|
---|
1011 | if (sqlcomb == "=") {
|
---|
1012 | // override what it means to do equality, to make it more like full text
|
---|
1013 | // searching
|
---|
1014 |
|
---|
1015 | text_t orterms = "";
|
---|
1016 | text_t term = "";
|
---|
1017 | bool in_phrase = false;
|
---|
1018 |
|
---|
1019 | text_t::const_iterator here = querystring.begin();
|
---|
1020 | text_t::const_iterator end = querystring.end();
|
---|
1021 | while (here != end) {
|
---|
1022 | if (is_unicode_letdig(*here)) {
|
---|
1023 | term.push_back(*here);
|
---|
1024 | }
|
---|
1025 | else if (*here == '"') {
|
---|
1026 | term.push_back(*here);
|
---|
1027 | if (!in_phrase) {
|
---|
1028 | in_phrase = true;
|
---|
1029 | } else {
|
---|
1030 | in_phrase = false;
|
---|
1031 | }
|
---|
1032 | }
|
---|
1033 | else if (in_phrase) {
|
---|
1034 | // Found word boundary, but in a phrase, so does not complete term
|
---|
1035 | term.push_back(*here);
|
---|
1036 | }
|
---|
1037 | else {
|
---|
1038 | // Found a word boundary
|
---|
1039 | if (!orterms.empty()) {
|
---|
1040 | orterms += " OR ";
|
---|
1041 | }
|
---|
1042 | orterms += "value LIKE '%" + term + "%'";
|
---|
1043 | term.clear();
|
---|
1044 | }
|
---|
1045 | ++here;
|
---|
1046 | }
|
---|
1047 |
|
---|
1048 | if (!term.empty()) {
|
---|
1049 | if (!orterms.empty()) {
|
---|
1050 | orterms += " OR ";
|
---|
1051 | }
|
---|
1052 | orterms += "value LIKE '%" + term + "%'";
|
---|
1053 | }
|
---|
1054 |
|
---|
1055 | element_in += orterms;
|
---|
1056 | }
|
---|
1057 | //We cast the value from STRING to REAL to allow numeric sorting
|
---|
1058 | else if (sqlcomb == "<num") {
|
---|
1059 | element_in += "CAST(value as REAL) < CAST('" + querystring+"' AS REAL)";
|
---|
1060 | }
|
---|
1061 | else if (sqlcomb == ">num") {
|
---|
1062 | element_in += "CAST(value as REAL) > CAST('" + querystring+"' AS REAL)";
|
---|
1063 | }
|
---|
1064 | else if (sqlcomb == "<=num") {
|
---|
1065 | element_in += "CAST(value as REAL) <= CAST('" + querystring+"' AS REAL)";
|
---|
1066 | }
|
---|
1067 | else if (sqlcomb == ">=num") {
|
---|
1068 | element_in += "CAST(value as REAL) >= CAST('" + querystring+"' AS REAL)";
|
---|
1069 | }
|
---|
1070 | else if (sqlcomb == "=num") {
|
---|
1071 | element_in += "CAST(value as REAL) = CAST('" + querystring+"' AS REAL)";
|
---|
1072 | }
|
---|
1073 | else {
|
---|
1074 | // search on value is "as is" querystring
|
---|
1075 | element_in += "value " + sqlcomb + " '" + querystring+"'";
|
---|
1076 | }
|
---|
1077 |
|
---|
1078 |
|
---|
1079 | querystring = element_in + "))";
|
---|
1080 |
|
---|
1081 | }
|
---|
1082 |
|
---|
1083 |
|
---|
1084 | void format_field_info_lucene(text_t &querystring, text_t &tag, int argt, int argb) {
|
---|
1085 |
|
---|
1086 | int type = 2; //lucene
|
---|
1087 |
|
---|
1088 | if (argb==0) { // simple
|
---|
1089 | // there will be no & or | as they should have already been removed
|
---|
1090 | // just tag the entire thing
|
---|
1091 | if (tag != "") {
|
---|
1092 | add_field_info(querystring, tag, type);
|
---|
1093 | }
|
---|
1094 | return;
|
---|
1095 | }
|
---|
1096 |
|
---|
1097 | // need to replace & with &&, | with ||
|
---|
1098 | text_t::const_iterator here = querystring.begin();
|
---|
1099 | text_t::const_iterator end = querystring.end();
|
---|
1100 |
|
---|
1101 | text_t finalquery = "";
|
---|
1102 | while (here != end) {
|
---|
1103 | if (*here == '&') {
|
---|
1104 | finalquery.push_back('&');
|
---|
1105 | finalquery.push_back('&');
|
---|
1106 | while (*(here+1) == '&') {
|
---|
1107 | ++here;
|
---|
1108 | }
|
---|
1109 | }
|
---|
1110 | else if (*here == '|') {
|
---|
1111 | finalquery.push_back('|');
|
---|
1112 | finalquery.push_back('|');
|
---|
1113 | while (*(here+1) == '|') {
|
---|
1114 | ++here;
|
---|
1115 | }
|
---|
1116 | }
|
---|
1117 | else {
|
---|
1118 | finalquery.push_back(*here);
|
---|
1119 | }
|
---|
1120 | ++here;
|
---|
1121 | }
|
---|
1122 | querystring = finalquery;
|
---|
1123 | add_field_info(querystring, tag, type);
|
---|
1124 | }
|
---|
1125 |
|
---|
1126 |
|
---|
1127 | void format_field_info_mgpp(text_t &querystring, text_t tag, int argt, int argb) {
|
---|
1128 |
|
---|
1129 | if (tag == "ZZ") tag = ""; // ZZ is a special tag meaning no tag (all fields)
|
---|
1130 | if (tag == "" && argb == 1) {
|
---|
1131 | return; // no field specifier, advanced mode, the query stays as written
|
---|
1132 | }
|
---|
1133 |
|
---|
1134 | int type = 1; // mgpp
|
---|
1135 |
|
---|
1136 | bool simple_and = (argb==0 && argt==0);
|
---|
1137 | text_t finalquery = "";
|
---|
1138 | text_t fieldpart ="";
|
---|
1139 | text_t queryelem = "";
|
---|
1140 | bool in_phrase = false;
|
---|
1141 | bool in_field = false;
|
---|
1142 |
|
---|
1143 | text_t::const_iterator here = querystring.begin();
|
---|
1144 | text_t::const_iterator end = querystring.end();
|
---|
1145 | while (here != end) {
|
---|
1146 | if (is_unicode_letdig(*here) || *here == '&' || is_special_character(type, *here)) {
|
---|
1147 | queryelem.push_back(*here);
|
---|
1148 | }
|
---|
1149 | else if (*here == '|') {
|
---|
1150 | in_field = false;
|
---|
1151 | }
|
---|
1152 | else if (*here == '!' || *here == '(' || *here == ')') {
|
---|
1153 | if (!in_phrase) { // ignore these if in_phrase
|
---|
1154 | // output field, then output operator
|
---|
1155 | in_field = false;
|
---|
1156 | if (!queryelem.empty()) {
|
---|
1157 | if (!simple_and && !fieldpart.empty()) {
|
---|
1158 | add_field_info(fieldpart, tag, type);
|
---|
1159 | finalquery += fieldpart;
|
---|
1160 | finalquery.push_back(' ');
|
---|
1161 | fieldpart.clear();
|
---|
1162 | }
|
---|
1163 | fieldpart += queryelem;
|
---|
1164 | }
|
---|
1165 | if (!fieldpart.empty()) {
|
---|
1166 | add_field_info(fieldpart, tag, type);
|
---|
1167 | finalquery += fieldpart;
|
---|
1168 | finalquery.push_back(' ');
|
---|
1169 | }
|
---|
1170 | fieldpart.clear();
|
---|
1171 | queryelem.clear();
|
---|
1172 | finalquery.push_back(*here);
|
---|
1173 | finalquery.push_back(' ');
|
---|
1174 | }
|
---|
1175 | }
|
---|
1176 | else if (*here == '"') {
|
---|
1177 | queryelem.push_back(*here);
|
---|
1178 | if (in_phrase == false) in_phrase = true;
|
---|
1179 | else {
|
---|
1180 | in_phrase = false;
|
---|
1181 | }
|
---|
1182 | }
|
---|
1183 |
|
---|
1184 | // Found word boundary, in a phrase
|
---|
1185 | else if (in_phrase) {
|
---|
1186 | queryelem.push_back(*here);
|
---|
1187 | }
|
---|
1188 | // Found a word boundary
|
---|
1189 | else {
|
---|
1190 | if (!queryelem.empty()) {
|
---|
1191 | if (queryelem == "&") {
|
---|
1192 | in_field = true;
|
---|
1193 | queryelem.clear();
|
---|
1194 | }
|
---|
1195 | else if (starts_with(queryelem, "NEAR") || starts_with(queryelem, "WITHIN")) {
|
---|
1196 |
|
---|
1197 | if (argb==1) {
|
---|
1198 | // simple search, these not allowed
|
---|
1199 | in_field = true;
|
---|
1200 | fieldpart += queryelem;
|
---|
1201 | fieldpart.push_back(' ');
|
---|
1202 | }
|
---|
1203 | queryelem.clear();
|
---|
1204 |
|
---|
1205 | }
|
---|
1206 | else {
|
---|
1207 | if (!simple_and && !in_field) {
|
---|
1208 | if (!fieldpart.empty()) {
|
---|
1209 | add_field_info(fieldpart, tag, type);
|
---|
1210 | finalquery += fieldpart;
|
---|
1211 | finalquery.push_back(' ');
|
---|
1212 | fieldpart.clear();
|
---|
1213 | }
|
---|
1214 | }
|
---|
1215 |
|
---|
1216 | fieldpart += queryelem;
|
---|
1217 | fieldpart.push_back(' ');
|
---|
1218 | queryelem.clear();
|
---|
1219 | }
|
---|
1220 | }
|
---|
1221 | }
|
---|
1222 | ++here;
|
---|
1223 | }
|
---|
1224 | // at the end
|
---|
1225 | if (!queryelem.empty()) {
|
---|
1226 | if (!simple_and && !in_field && !fieldpart.empty()) {
|
---|
1227 | add_field_info(fieldpart, tag, type);
|
---|
1228 | finalquery += fieldpart;
|
---|
1229 | finalquery.push_back(' ');
|
---|
1230 | fieldpart.clear();
|
---|
1231 | }
|
---|
1232 | fieldpart += queryelem;
|
---|
1233 | }
|
---|
1234 | if (!fieldpart.empty()) {
|
---|
1235 | add_field_info(fieldpart, tag, type);
|
---|
1236 | finalquery += fieldpart;
|
---|
1237 | fieldpart.clear();
|
---|
1238 |
|
---|
1239 | // doesn't the following just leave a dangling space at the end ?? (used to make mgpp crash)
|
---|
1240 | // consider cutting this line
|
---|
1241 | finalquery.push_back(' ');
|
---|
1242 | }
|
---|
1243 |
|
---|
1244 | querystring = finalquery;
|
---|
1245 | }
|
---|
1246 |
|
---|
1247 |
|
---|
1248 | void format_field_info_sql(text_t &querystring, const text_t &tagseq,
|
---|
1249 | const text_t &sqlcomb,
|
---|
1250 | int argt, int argb)
|
---|
1251 | {
|
---|
1252 | add_field_info_sql(querystring, tagseq, sqlcomb);
|
---|
1253 | }
|
---|
1254 |
|
---|
1255 |
|
---|
1256 | void format_field_info(text_t &querystring, text_t tag, int argct, int argt, int argb) {
|
---|
1257 | if (argct == 1) {
|
---|
1258 | format_field_info_mgpp(querystring, tag, argt, argb);
|
---|
1259 | } else if (argct == 2) {
|
---|
1260 | format_field_info_lucene(querystring, tag, argt, argb);
|
---|
1261 | }
|
---|
1262 | }
|
---|
1263 |
|
---|
1264 | void mgpp_adddateelem(text_t& querystring, const int date)
|
---|
1265 | {
|
---|
1266 | querystring.appendcstr(" [");
|
---|
1267 | if(date<0) {
|
---|
1268 | querystring.appendcstr("bc");
|
---|
1269 | querystring.appendint((date*-1));
|
---|
1270 | }
|
---|
1271 | else {
|
---|
1272 | querystring.appendint(date);
|
---|
1273 | }
|
---|
1274 | querystring.appendcstr("]:CV");
|
---|
1275 | }
|
---|
1276 |
|
---|
1277 | void lucene_adddateelem(text_t& querystring, const int date)
|
---|
1278 | {
|
---|
1279 | querystring.appendcstr(" CV:(");
|
---|
1280 | if(date<0) {
|
---|
1281 | querystring.appendcstr("bc");
|
---|
1282 | querystring.appendint((date*-1));
|
---|
1283 | }
|
---|
1284 | else {
|
---|
1285 | querystring.appendint(date);
|
---|
1286 | }
|
---|
1287 | querystring.appendcstr(")");
|
---|
1288 | }
|
---|
1289 |
|
---|
1290 |
|
---|
1291 | void add_dates(text_t &querystring, int startdate, int enddate,
|
---|
1292 | int startbc, int endbc, int ct)
|
---|
1293 | {
|
---|
1294 | if(startdate)
|
---|
1295 | {
|
---|
1296 | int querystringis = 0;
|
---|
1297 | text_t::const_iterator here = querystring.begin();
|
---|
1298 | text_t::const_iterator end = querystring.end();
|
---|
1299 | while(here!=end)
|
---|
1300 | {
|
---|
1301 | if(!(isspace((*here)))){
|
---|
1302 | here = end;
|
---|
1303 | querystringis = 1;
|
---|
1304 | }
|
---|
1305 | else
|
---|
1306 | ++here;
|
---|
1307 | }
|
---|
1308 | //converting BCE dates
|
---|
1309 | if(startbc && startdate > 0)
|
---|
1310 | {
|
---|
1311 | startdate *= -1;
|
---|
1312 | }
|
---|
1313 | if(endbc && enddate > 0)
|
---|
1314 | {
|
---|
1315 | enddate *= -1;
|
---|
1316 | }
|
---|
1317 | if(enddate != 0 && enddate<startdate)
|
---|
1318 | {
|
---|
1319 | cout<<"enddate too small"<<endl;
|
---|
1320 | return;
|
---|
1321 | }
|
---|
1322 | if(querystringis)
|
---|
1323 | querystring.appendcstr(" AND");
|
---|
1324 | if(!enddate)
|
---|
1325 | {
|
---|
1326 | if (ct==1) {
|
---|
1327 | mgpp_adddateelem(querystring,startdate);
|
---|
1328 | }
|
---|
1329 | else { // lucene
|
---|
1330 | lucene_adddateelem(querystring,startdate);
|
---|
1331 | }
|
---|
1332 | }
|
---|
1333 | else{
|
---|
1334 | int nextdate = startdate;
|
---|
1335 | querystring.appendcstr(" (");
|
---|
1336 | while(nextdate<=enddate)
|
---|
1337 | {
|
---|
1338 | if(nextdate!=0) {
|
---|
1339 | if (ct==1) {
|
---|
1340 | mgpp_adddateelem(querystring,nextdate);
|
---|
1341 | }
|
---|
1342 | else { // lucene
|
---|
1343 | lucene_adddateelem(querystring,nextdate);
|
---|
1344 | }
|
---|
1345 | }
|
---|
1346 | ++nextdate;
|
---|
1347 | }
|
---|
1348 | querystring.appendcstr(" )");
|
---|
1349 | }
|
---|
1350 | }
|
---|
1351 |
|
---|
1352 | }
|
---|