Skip to content

Commit d226398

Browse files
rouiljrouilj
committed
issue2551189 - increase text search maxlength
This removes I think all the magic references to 25 and 30 (varchar size) and replaces them with references to maxlength or maxlength+5. I am not sure why the db column is 5 characters larger than the size of what should be the max size of a word, but I'll keep the buffer of 5 as making it 1/5 the size of maxlength makes less sense. Also added tests for fts search in templating which were missing. Added postgres, mysql and sqlite native indexing backends in which to test fts. Added fts test to native-fts as well to make sure it's working. I want to commit this now for CI. Todo: add test cases for the use of FTS in the csv output in actions.py. There is no test coverage of the match case there. change maxlength to a higher value (50) as requested in the ticket. Modify existing extremewords test cases to allow words > 25 and < 51 write code to migrate column sizes for mysql and postgresql to match maxlength I will roll this into the version 7 schema update that supports use of database fts support.
1 parent 09176fa commit d226398

File tree

5 files changed

+183
-18
lines changed

5 files changed

+183
-18
lines changed

‎roundup/backends/back_mysql.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -234,8 +234,10 @@ def create_version_2_tables(self):
234234
self.sql('''CREATE TABLE __textids (_class VARCHAR(255),
235235
_itemid VARCHAR(255), _prop VARCHAR(255), _textid INT)
236236
ENGINE=%s'''%self.mysql_backend)
237-
self.sql('''CREATE TABLE __words (_word VARCHAR(30),
238-
_textid INT) ENGINE=%s'''%self.mysql_backend)
237+
self.sql('''CREATE TABLE __words (_word VARCHAR(%s),
238+
_textid INT) ENGINE=%s''' % ((self.indexer.maxlength + 5),
239+
self.mysql_backend)
240+
)
239241
self.sql('CREATE INDEX words_word_ids ON __words(_word)')
240242
self.sql('CREATE INDEX words_by_id ON __words (_textid)')
241243
self.sql('CREATE UNIQUE INDEX __textids_by_props ON '

‎roundup/backends/back_postgresql.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -235,8 +235,8 @@ def create_version_2_tables(self):
235235
self.sql('''CREATE TABLE __textids (
236236
_textid integer primary key, _class VARCHAR(255),
237237
_itemid VARCHAR(255), _prop VARCHAR(255))''')
238-
self.sql('''CREATE TABLE __words (_word VARCHAR(30),
239-
_textid integer)''')
238+
self.sql('''CREATE TABLE __words (_word VARCHAR(%s),
239+
_textid integer)''' % (self.indexer.maxlength + 5))
240240
self.sql('CREATE INDEX words_word_idx ON __words(_word)')
241241
self.sql('CREATE INDEX words_by_id ON __words (_textid)')
242242
self.sql('CREATE UNIQUE INDEX __textids_by_props ON '

‎roundup/cgi/actions.py‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1433,9 +1433,10 @@ def handle(self):
14331433

14341434
# full-text search
14351435
if request.search_text:
1436+
indexer = self.db.indexer
14361437
if self.db.indexer.query_language:
14371438
try:
1438-
matches = self.db.indexer.search(
1439+
matches = indexer.search(
14391440
[request.search_text], klass)
14401441
except Exception as e:
14411442
error = " ".join(e.args)
@@ -1444,8 +1445,10 @@ def handle(self):
14441445
# trigger error reporting. NotFound isn't right but...
14451446
raise exceptions.NotFound(error)
14461447
else:
1447-
matches = self.db.indexer.search(
1448-
re.findall(r'\b\w{2,25}\b', request.search_text), klass)
1448+
matches = indexer.search(
1449+
re.findall(r'\b\w{%s,%s}\b' % (indexer.minlength,
1450+
indexer.maxlength),
1451+
request.search_text), klass)
14491452
else:
14501453
matches = None
14511454

@@ -1609,9 +1612,10 @@ def handle(self):
16091612

16101613
# full-text search
16111614
if request.search_text:
1612-
if self.db.indexer.query_language:
1615+
indexer = self.db.indexer
1616+
if indexer.query_language:
16131617
try:
1614-
matches = self.db.indexer.search(
1618+
matches = indexer.search(
16151619
[request.search_text], klass)
16161620
except Exception as e:
16171621
error = " ".join(e.args)
@@ -1620,8 +1624,10 @@ def handle(self):
16201624
# trigger error reporting. NotFound isn't right but...
16211625
raise exceptions.NotFound(error)
16221626
else:
1623-
matches = self.db.indexer.search(
1624-
re.findall(r'\b\w{2,25}\b', request.search_text),
1627+
matches = indexer.search(
1628+
re.findall(r'\b\w{%s,%s}\b' % (indexer.minlength,
1629+
indexer.maxlength),
1630+
request.search_text),
16251631
klass)
16261632
else:
16271633
matches = None

‎roundup/cgi/templating.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3318,17 +3318,19 @@ def batch(self, permission='View'):
33183318
# get the list of ids we're batching over
33193319
klass = self.client.db.getclass(self.classname)
33203320
if self.search_text:
3321-
if self.client.db.indexer.query_language:
3321+
indexer = self.client.db.indexer
3322+
if indexer.query_language:
33223323
try:
3323-
matches = self.client.db.indexer.search(
3324+
matches = indexer.search(
33243325
[self.search_text], klass)
33253326
except Exception as e:
33263327
self.client.add_error_message(" ".join(e.args))
33273328
raise
33283329
else:
3329-
matches = self.client.db.indexer.search(
3330+
matches = indexer.search(
33303331
[u2s(w.upper()) for w in re.findall(
3331-
r'(?u)\b\w{2,25}\b',
3332+
r'(?u)\b\w{%s,%s}\b' % (indexer.minlength,
3333+
indexer.maxlength),
33323334
s2u(self.search_text, "replace")
33333335
)], klass)
33343336
else:

‎test/test_cgi.py‎

Lines changed: 158 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@
3333
from . import db_test_base
3434
from .db_test_base import FormTestParent, setupTracker, FileUpload
3535
from .cmp_helper import StringFragmentCmpHelper
36+
from .test_postgresql import skip_postgresql
37+
from .test_mysql import skip_mysql
38+
3639

3740
class FileList:
3841
def __init__(self, name, *files):
@@ -42,6 +45,24 @@ def items (self):
4245
for f in self.files:
4346
yield (self.name, f)
4447

48+
class testFtsQuery(object):
49+
50+
def testRenderContextFtsQuery(self):
51+
self.db.issue.create(title='i1 is found', status="chatting")
52+
53+
self.client.form=db_test_base.makeForm(
54+
{ "@ok_message": "ok message", "@template": "index",
55+
"@search_text": "found"})
56+
self.client.path = 'issue'
57+
self.client.determine_context()
58+
59+
result = self.client.renderContext()
60+
61+
expected = '">i1 is found</a>'
62+
63+
self.assertIn(expected, result)
64+
self.assertEqual(self.client.response_code, 200)
65+
4566
cm = client.add_message
4667
class MessageTestCase(unittest.TestCase):
4768
# Note: Escaping is now handled on a message-by-message basis at a
@@ -1976,7 +1997,7 @@ def testCSVExportWithIdFailPermissionValidColumn(self):
19761997
self.assertRaises(exceptions.Unauthorised,
19771998
actions.ExportCSVWithIdAction(cl).handle)
19781999

1979-
class TemplateHtmlRendering(unittest.TestCase):
2000+
class TemplateHtmlRendering(unittest.TestCase, testFtsQuery):
19802001
''' try to test the rendering code for tal '''
19812002
def setUp(self):
19822003
self.dirname = '_test_template'
@@ -2343,9 +2364,10 @@ def testTemplateSubdirectory(self):
23432364
r = t.selectTemplate("user", "subdir/item")
23442365
self.assertEqual("subdir/user.item", r)
23452366

2346-
class SqliteCgiTest(unittest.TestCase):
2367+
class SqliteNativeFtsCgiTest(unittest.TestCase, testFtsQuery):
23472368
"""All of the rest of the tests use anydbm as the backend.
2348-
This class tests renderError when renderContext fails.
2369+
In addtion to normal fts test, this class tests renderError
2370+
when renderContext fails.
23492371
Triggering this error requires the native-fts backend for
23502372
the sqlite db.
23512373
"""
@@ -2407,5 +2429,138 @@ def testRenderContextBadFtsQuery(self):
24072429
self.assertEqual(result, expected)
24082430
self.assertEqual(self.client.response_code, 400)
24092431

2432+
class SqliteNativeCgiTest(unittest.TestCase, testFtsQuery):
2433+
"""All of the rest of the tests use anydbm as the backend.
2434+
This class tests renderContext for fulltext search.
2435+
Run with sqlite and native indexer.
2436+
"""
2437+
2438+
def setUp(self):
2439+
self.dirname = '_test_template'
2440+
# set up and open a tracker
2441+
self.instance = setupTracker(self.dirname, backend="sqlite")
2442+
2443+
self.instance.config.INDEXER = "native"
2444+
2445+
# open the database
2446+
self.db = self.instance.open('admin')
2447+
self.db.tx_Source = "web"
2448+
2449+
# create a client instance and hijack write_html
2450+
self.client = client.Client(self.instance, "user",
2451+
{'PATH_INFO':'/user', 'REQUEST_METHOD':'POST'},
2452+
form=db_test_base.makeForm({"@template": "item"}))
2453+
2454+
self.client._error_message = []
2455+
self.client._ok_message = []
2456+
self.client.db = self.db
2457+
self.client.userid = '1'
2458+
self.client.language = ('en',)
2459+
self.client.session_api = MockNull(_sid="1234567890")
2460+
2461+
self.output = []
2462+
# ugly hack to get html_write to return data here.
2463+
def html_write(s):
2464+
self.output.append(s)
2465+
2466+
# hijack html_write
2467+
self.client.write_html = html_write
2468+
2469+
def tearDown(self):
2470+
self.db.close()
2471+
try:
2472+
shutil.rmtree(self.dirname)
2473+
except OSError as error:
2474+
if error.errno not in (errno.ENOENT, errno.ESRCH): raise
2475+
2476+
@skip_postgresql
2477+
class PostgresqlNativeCgiTest(unittest.TestCase, testFtsQuery):
2478+
"""All of the rest of the tests use anydbm as the backend.
2479+
This class tests renderContext for fulltext search.
2480+
Run with postgresql and native indexer.
2481+
"""
2482+
2483+
def setUp(self):
2484+
self.dirname = '_test_template'
2485+
# set up and open a tracker
2486+
self.instance = setupTracker(self.dirname, backend="postgresql")
2487+
2488+
self.instance.config.INDEXER = "native"
2489+
2490+
# open the database
2491+
self.db = self.instance.open('admin')
2492+
self.db.tx_Source = "web"
2493+
2494+
# create a client instance and hijack write_html
2495+
self.client = client.Client(self.instance, "user",
2496+
{'PATH_INFO':'/user', 'REQUEST_METHOD':'POST'},
2497+
form=db_test_base.makeForm({"@template": "item"}))
2498+
2499+
self.client._error_message = []
2500+
self.client._ok_message = []
2501+
self.client.db = self.db
2502+
self.client.userid = '1'
2503+
self.client.language = ('en',)
2504+
self.client.session_api = MockNull(_sid="1234567890")
2505+
2506+
self.output = []
2507+
# ugly hack to get html_write to return data here.
2508+
def html_write(s):
2509+
self.output.append(s)
2510+
2511+
# hijack html_write
2512+
self.client.write_html = html_write
2513+
2514+
def tearDown(self):
2515+
self.db.close()
2516+
try:
2517+
shutil.rmtree(self.dirname)
2518+
except OSError as error:
2519+
if error.errno not in (errno.ENOENT, errno.ESRCH): raise
2520+
2521+
@skip_mysql
2522+
class MysqlNativeCgiTest(unittest.TestCase, testFtsQuery):
2523+
"""All of the rest of the tests use anydbm as the backend.
2524+
This class tests renderContext for fulltext search.
2525+
Run with mysql and native indexer.
2526+
"""
2527+
2528+
def setUp(self):
2529+
self.dirname = '_test_template'
2530+
# set up and open a tracker
2531+
self.instance = setupTracker(self.dirname, backend="mysql")
2532+
2533+
self.instance.config.INDEXER = "native"
2534+
2535+
# open the database
2536+
self.db = self.instance.open('admin')
2537+
self.db.tx_Source = "web"
2538+
2539+
# create a client instance and hijack write_html
2540+
self.client = client.Client(self.instance, "user",
2541+
{'PATH_INFO':'/user', 'REQUEST_METHOD':'POST'},
2542+
form=db_test_base.makeForm({"@template": "item"}))
2543+
2544+
self.client._error_message = []
2545+
self.client._ok_message = []
2546+
self.client.db = self.db
2547+
self.client.userid = '1'
2548+
self.client.language = ('en',)
2549+
self.client.session_api = MockNull(_sid="1234567890")
2550+
2551+
self.output = []
2552+
# ugly hack to get html_write to return data here.
2553+
def html_write(s):
2554+
self.output.append(s)
2555+
2556+
# hijack html_write
2557+
self.client.write_html = html_write
2558+
2559+
def tearDown(self):
2560+
self.db.close()
2561+
try:
2562+
shutil.rmtree(self.dirname)
2563+
except OSError as error:
2564+
if error.errno not in (errno.ENOENT, errno.ESRCH): raise
24102565

24112566
# vim: set filetype=python sts=4 sw=4 et si :

0 commit comments

Comments
 (0)