From d2ea3a6c19a3729b830bf2a3069b47dfc9b54e7b Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 20 Oct 2017 23:23:16 +0200 Subject: [PATCH 1/4] case-insensitive search for non-ASCII --- cps/db.py | 62 +++++++++++++++++++++------------------ cps/web.py | 41 ++++++++++++++------------ optional-requirements.txt | 2 +- 3 files changed, 57 insertions(+), 48 deletions(-) diff --git a/cps/db.py b/cps/db.py index f337ea4c..8be77977 100755 --- a/cps/db.py +++ b/cps/db.py @@ -26,43 +26,45 @@ def title_sort(title): title = title.replace(prep, '') + ', ' + prep return title.strip() +def lowercase(string): + return string.lower() Base = declarative_base() books_authors_link = Table('books_authors_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('author', Integer, ForeignKey('authors.id'), primary_key=True) + Column('book', Integer, ForeignKey('books.id'), primary_key = True), + Column('author', Integer, ForeignKey('authors.id'), primary_key = True) ) books_tags_link = Table('books_tags_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('tag', Integer, ForeignKey('tags.id'), primary_key=True) + Column('book', Integer, ForeignKey('books.id'), primary_key = True), + Column('tag', Integer, ForeignKey('tags.id'), primary_key = True) ) books_series_link = Table('books_series_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('series', Integer, ForeignKey('series.id'), primary_key=True) + Column('book', Integer, ForeignKey('books.id'), primary_key = True), + Column('series', Integer, ForeignKey('series.id'), primary_key = True) ) books_ratings_link = Table('books_ratings_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('rating', Integer, ForeignKey('ratings.id'), primary_key=True) + Column('book', Integer, ForeignKey('books.id'), primary_key = True), + Column('rating', Integer, ForeignKey('ratings.id'), primary_key = True) ) books_languages_link = Table('books_languages_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('lang_code', Integer, ForeignKey('languages.id'), primary_key=True) + Column('book', Integer, ForeignKey('books.id'), primary_key = True), + Column('lang_code', Integer, ForeignKey('languages.id'), primary_key = True) ) books_publishers_link = Table('books_publishers_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('publisher', Integer, ForeignKey('publishers.id'), primary_key=True) + Column('book', Integer, ForeignKey('books.id'), primary_key = True), + Column('publisher', Integer, ForeignKey('publishers.id'), primary_key = True) ) class Identifiers(Base): __tablename__ = 'identifiers' - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key = True) type = Column(String) val = Column(String) book = Column(Integer, ForeignKey('books.id')) @@ -110,7 +112,7 @@ class Identifiers(Base): class Comments(Base): __tablename__ = 'comments' - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key = True) text = Column(String) book = Column(Integer, ForeignKey('books.id')) @@ -125,8 +127,8 @@ class Comments(Base): class Tags(Base): __tablename__ = 'tags' - id = Column(Integer, primary_key=True, autoincrement=True) - name = Column(String) + id = Column(Integer, primary_key = True, autoincrement = True) + name = Column(String(collation = 'NOCASE')) def __init__(self, name): self.name = name @@ -138,8 +140,8 @@ class Tags(Base): class Authors(Base): __tablename__ = 'authors' - id = Column(Integer, primary_key=True) - name = Column(String) + id = Column(Integer, primary_key = True) + name = Column(String(collation = 'U_NOCASE')) sort = Column(String) link = Column(String) @@ -155,8 +157,8 @@ class Authors(Base): class Series(Base): __tablename__ = 'series' - id = Column(Integer, primary_key=True) - name = Column(String) + id = Column(Integer, primary_key = True) + name = Column(String(collation = 'U_NOCASE')) sort = Column(String) def __init__(self, name, sort): @@ -170,7 +172,7 @@ class Series(Base): class Ratings(Base): __tablename__ = 'ratings' - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key = True) rating = Column(Integer) def __init__(self, rating): @@ -183,7 +185,7 @@ class Ratings(Base): class Languages(Base): __tablename__ = 'languages' - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key = True) lang_code = Column(String) def __init__(self, lang_code): @@ -195,8 +197,8 @@ class Languages(Base): class Publishers(Base): __tablename__ = 'publishers' - id = Column(Integer, primary_key=True) - name = Column(String) + id = Column(Integer, primary_key = True) + name = Column(String(collation = 'U_NOCASE')) sort = Column(String) def __init__(self, name,sort): @@ -211,7 +213,7 @@ class Publishers(Base): class Data(Base): __tablename__ = 'data' - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key = True) book = Column(Integer, ForeignKey('books.id')) format = Column(String) uncompressed_size = Column(Integer) @@ -232,8 +234,8 @@ class Books(Base): DEFAULT_PUBDATE = "0101-01-01 00:00:00+00:00" - id = Column(Integer, primary_key=True) - title = Column(String) + id = Column(Integer, primary_key = True) + title = Column(String(collation = 'U_NOCASE')) sort = Column(String) author_sort = Column(String) timestamp = Column(String) @@ -275,7 +277,7 @@ class Books(Base): class Custom_Columns(Base): __tablename__ = 'custom_columns' - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key = True) label = Column(String) name = Column(String) datatype = Column(String) @@ -294,6 +296,7 @@ def setup_db(): global engine global session global cc_classes + global conn if config.config_calibre_dir is None or config.config_calibre_dir == u'': content = ub.session.query(ub.Settings).first() @@ -319,6 +322,7 @@ def setup_db(): ub.session.commit() config.loadSettings() conn.connection.create_function('title_sort', 1, title_sort) + conn.connection.create_function('lower', 1, lowercase) if not cc_classes: cc = conn.execute("SELECT id, datatype FROM custom_columns") @@ -365,6 +369,6 @@ def setup_db(): # Base.metadata.create_all(engine) Session = sessionmaker() - Session.configure(bind=engine) + Session.configure(bind=conn) session = Session() return True diff --git a/cps/web.py b/cps/web.py index ef44328d..ff822b3e 100755 --- a/cps/web.py +++ b/cps/web.py @@ -659,11 +659,11 @@ def feed_normal_search(): def feed_search(term): if term: - entries = db.session.query(db.Books).filter(db.or_(db.Books.tags.any(db.Tags.name.like("%" + term + "%")), - db.Books.series.any(db.Series.name.like("%" + term + "%")), - db.Books.authors.any(db.Authors.name.like("%" + term + "%")), - db.Books.publishers.any(db.Publishers.name.like("%" + term + "%")), - db.Books.title.like("%" + term + "%")))\ + entries = db.session.query(db.Books).filter(db.or_(db.Books.tags.any(db.Tags.name.ilike("%" + term + "%")), + db.Books.series.any(db.Series.name.ilike("%" + term + "%")), + db.Books.authors.any(db.Authors.name.ilike("%" + term + "%")), + db.Books.publishers.any(db.Publishers.name.ilike("%" + term + "%")), + db.Books.title.ilike("%" + term + "%")))\ .filter(common_filters()).all() entriescount = len(entries) if len(entries) > 0 else 1 pagination = Pagination(1, entriescount, entriescount) @@ -909,7 +909,7 @@ def get_authors_json(): if request.method == "GET": query = request.args.get('q') # entries = db.session.execute("select name from authors where name like '%" + query + "%'") - entries = db.session.query(db.Authors).filter(db.Authors.name.like("%" + query + "%")).all() + entries = db.session.query(db.Authors).filter(db.Authors.name.ilike("%" + query + "%")).all() json_dumps = json.dumps([dict(name=r.name) for r in entries]) return json_dumps @@ -920,7 +920,7 @@ def get_tags_json(): if request.method == "GET": query = request.args.get('q') # entries = db.session.execute("select name from tags where name like '%" + query + "%'") - entries = db.session.query(db.Tags).filter(db.Tags.name.like("%" + query + "%")).all() + entries = db.session.query(db.Tags).filter(db.Tags.iname.ilike("%" + query + "%")).all() # for x in entries: # alfa = dict(name=x.name) json_dumps = json.dumps([dict(name=r.name) for r in entries]) @@ -998,7 +998,7 @@ def get_languages_json(): def get_series_json(): if request.method == "GET": query = request.args.get('q') - entries = db.session.query(db.Series).filter(db.Series.name.like("%" + query + "%")).all() + entries = db.session.query(db.Series).filter(db.Series.name.ilike("%" + query + "%")).all() # entries = db.session.execute("select name from series where name like '%" + query + "%'") json_dumps = json.dumps([dict(name=r.name) for r in entries]) return json_dumps @@ -1014,8 +1014,8 @@ def get_matching_tags(): title_input = request.args.get('book_title') include_tag_inputs = request.args.getlist('include_tag') exclude_tag_inputs = request.args.getlist('exclude_tag') - q = q.filter(db.Books.authors.any(db.Authors.name.like("%" + author_input + "%")), - db.Books.title.like("%" + title_input + "%")) + q = q.filter(db.Books.authors.any(db.Authors.name.ilike("%" + author_input + "%")), + db.Books.title.ilike("%" + title_input + "%")) if len(include_tag_inputs) > 0: for tag in include_tag_inputs: q = q.filter(db.Books.tags.any(db.Tags.id == tag)) @@ -1577,13 +1577,18 @@ def update(): @login_required_if_no_ano def search(): term = request.args.get("query").strip() + if term: - entries = db.session.query(db.Books).filter(db.or_(db.Books.tags.any(db.Tags.name.like("%" + term + "%")), - db.Books.series.any(db.Series.name.like("%" + term + "%")), - db.Books.authors.any(db.Authors.name.like("%" + term + "%")), - db.Books.publishers.any(db.Publishers.name.like("%" + term + "%")), - db.Books.title.like("%" + term + "%")))\ + entries = db.session.query(db.Books).filter(db.or_(db.Books.tags.any(db.Tags.name.ilike("%" + term + "%")), + db.Books.series.any(db.Series.name.ilike("%" + term + "%")), + db.Books.authors.any(db.Authors.name.ilike("%" + term + "%")), + db.Books.publishers.any(db.Publishers.name.ilike("%" + term + "%")), + db.Books.title.ilike("%" + term + "%")))\ .filter(common_filters()).all() +# entries = db.session.query(db.Books).with_entities(db.Books.title).filter(db.Books.title.ilike("%" + term + "%")).all() + #result = db.session.execute("select name from authors where lower(name) like '%" + term.lower() + "%'") + #entries = result.fetchall() + #result.close() return render_title_template('search.html', searchterm=term, entries=entries) else: return render_title_template('search.html', searchterm="") @@ -1625,9 +1630,9 @@ def advanced_search(): lang.name = _(isoLanguages.get(part3=lang.lang_code).name) searchterm.extend(language.name for language in language_names) searchterm = " + ".join(filter(None, searchterm)) - q = q.filter(db.Books.authors.any(db.Authors.name.like("%" + author_name + "%")), - db.Books.title.like("%" + book_title + "%"), - db.Books.publishers.any(db.Publishers.name.like("%" + publisher + "%"))) + q = q.filter(db.Books.authors.any(db.Authors.name.ilike("%" + author_name + "%")), + db.Books.title.ilike("%" + book_title + "%"), + db.Books.publishers.any(db.Publishers.name.ilike("%" + publisher + "%"))) for tag in include_tag_inputs: q = q.filter(db.Books.tags.any(db.Tags.id == tag)) for tag in exclude_tag_inputs: diff --git a/optional-requirements.txt b/optional-requirements.txt index cf743dbb..7a2ad451 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -12,4 +12,4 @@ rsa==3.4.2 six==1.10.0 uritemplate==3.0.0 goodreads>=0.3.2 -python-Levenshtein>=0.12.0 +#python-Levenshtein>=0.12.0 From 4a04ec898e0a61cac753246a3d295ed476341688 Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 21 Oct 2017 21:50:47 +0200 Subject: [PATCH 2/4] case-insensitive search for non-ASCII - improved --- cps/db.py | 69 ++++++++++++++++++++++++++++-------------------------- cps/web.py | 5 +++- 2 files changed, 40 insertions(+), 34 deletions(-) diff --git a/cps/db.py b/cps/db.py index 8be77977..7428eba1 100755 --- a/cps/db.py +++ b/cps/db.py @@ -26,45 +26,48 @@ def title_sort(title): title = title.replace(prep, '') + ', ' + prep return title.strip() -def lowercase(string): - return string.lower() +def lcase(s): + return s.lower() + +def ucase(s): + return s.upper() Base = declarative_base() books_authors_link = Table('books_authors_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key = True), - Column('author', Integer, ForeignKey('authors.id'), primary_key = True) + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('author', Integer, ForeignKey('authors.id'), primary_key=True) ) books_tags_link = Table('books_tags_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key = True), - Column('tag', Integer, ForeignKey('tags.id'), primary_key = True) + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('tag', Integer, ForeignKey('tags.id'), primary_key=True) ) books_series_link = Table('books_series_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key = True), - Column('series', Integer, ForeignKey('series.id'), primary_key = True) + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('series', Integer, ForeignKey('series.id'), primary_key=True) ) books_ratings_link = Table('books_ratings_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key = True), - Column('rating', Integer, ForeignKey('ratings.id'), primary_key = True) + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('rating', Integer, ForeignKey('ratings.id'), primary_key=True) ) books_languages_link = Table('books_languages_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key = True), - Column('lang_code', Integer, ForeignKey('languages.id'), primary_key = True) + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('lang_code', Integer, ForeignKey('languages.id'), primary_key=True) ) books_publishers_link = Table('books_publishers_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key = True), - Column('publisher', Integer, ForeignKey('publishers.id'), primary_key = True) + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('publisher', Integer, ForeignKey('publishers.id'), primary_key=True) ) class Identifiers(Base): __tablename__ = 'identifiers' - id = Column(Integer, primary_key = True) + id = Column(Integer, primary_key=True) type = Column(String) val = Column(String) book = Column(Integer, ForeignKey('books.id')) @@ -112,7 +115,7 @@ class Identifiers(Base): class Comments(Base): __tablename__ = 'comments' - id = Column(Integer, primary_key = True) + id = Column(Integer, primary_key=True) text = Column(String) book = Column(Integer, ForeignKey('books.id')) @@ -127,8 +130,8 @@ class Comments(Base): class Tags(Base): __tablename__ = 'tags' - id = Column(Integer, primary_key = True, autoincrement = True) - name = Column(String(collation = 'NOCASE')) + id = Column(Integer, primary_key=True, autoincrement=True) + name = Column(String) def __init__(self, name): self.name = name @@ -140,8 +143,8 @@ class Tags(Base): class Authors(Base): __tablename__ = 'authors' - id = Column(Integer, primary_key = True) - name = Column(String(collation = 'U_NOCASE')) + id = Column(Integer, primary_key=True) + name = Column(String) sort = Column(String) link = Column(String) @@ -157,8 +160,8 @@ class Authors(Base): class Series(Base): __tablename__ = 'series' - id = Column(Integer, primary_key = True) - name = Column(String(collation = 'U_NOCASE')) + id = Column(Integer, primary_key=True) + name = Column(String) sort = Column(String) def __init__(self, name, sort): @@ -172,7 +175,7 @@ class Series(Base): class Ratings(Base): __tablename__ = 'ratings' - id = Column(Integer, primary_key = True) + id = Column(Integer, primary_key=True) rating = Column(Integer) def __init__(self, rating): @@ -185,7 +188,7 @@ class Ratings(Base): class Languages(Base): __tablename__ = 'languages' - id = Column(Integer, primary_key = True) + id = Column(Integer, primary_key=True) lang_code = Column(String) def __init__(self, lang_code): @@ -197,8 +200,8 @@ class Languages(Base): class Publishers(Base): __tablename__ = 'publishers' - id = Column(Integer, primary_key = True) - name = Column(String(collation = 'U_NOCASE')) + id = Column(Integer, primary_key=True) + name = Column(String) sort = Column(String) def __init__(self, name,sort): @@ -213,7 +216,7 @@ class Publishers(Base): class Data(Base): __tablename__ = 'data' - id = Column(Integer, primary_key = True) + id = Column(Integer, primary_key=True) book = Column(Integer, ForeignKey('books.id')) format = Column(String) uncompressed_size = Column(Integer) @@ -234,8 +237,8 @@ class Books(Base): DEFAULT_PUBDATE = "0101-01-01 00:00:00+00:00" - id = Column(Integer, primary_key = True) - title = Column(String(collation = 'U_NOCASE')) + id = Column(Integer, primary_key=True) + title = Column(String) sort = Column(String) author_sort = Column(String) timestamp = Column(String) @@ -277,7 +280,7 @@ class Books(Base): class Custom_Columns(Base): __tablename__ = 'custom_columns' - id = Column(Integer, primary_key = True) + id = Column(Integer, primary_key=True) label = Column(String) name = Column(String) datatype = Column(String) @@ -296,7 +299,6 @@ def setup_db(): global engine global session global cc_classes - global conn if config.config_calibre_dir is None or config.config_calibre_dir == u'': content = ub.session.query(ub.Settings).first() @@ -322,7 +324,8 @@ def setup_db(): ub.session.commit() config.loadSettings() conn.connection.create_function('title_sort', 1, title_sort) - conn.connection.create_function('lower', 1, lowercase) + conn.connection.create_function('lower', 1, lcase) + conn.connection.create_function('upper', 1, ucase) if not cc_classes: cc = conn.execute("SELECT id, datatype FROM custom_columns") @@ -369,6 +372,6 @@ def setup_db(): # Base.metadata.create_all(engine) Session = sessionmaker() - Session.configure(bind=conn) + Session.configure(bind=engine) session = Session() return True diff --git a/cps/web.py b/cps/web.py index ff822b3e..feccd092 100755 --- a/cps/web.py +++ b/cps/web.py @@ -659,6 +659,7 @@ def feed_normal_search(): def feed_search(term): if term: + db.session.connection().connection.connection.create_function("lower", 1, db.lcase) entries = db.session.query(db.Books).filter(db.or_(db.Books.tags.any(db.Tags.name.ilike("%" + term + "%")), db.Books.series.any(db.Series.name.ilike("%" + term + "%")), db.Books.authors.any(db.Authors.name.ilike("%" + term + "%")), @@ -1576,9 +1577,10 @@ def update(): @app.route("/search", methods=["GET"]) @login_required_if_no_ano def search(): - term = request.args.get("query").strip() + term = request.args.get("query").strip().lower() if term: + db.session.connection().connection.connection.create_function("lower", 1, db.lcase) entries = db.session.query(db.Books).filter(db.or_(db.Books.tags.any(db.Tags.name.ilike("%" + term + "%")), db.Books.series.any(db.Series.name.ilike("%" + term + "%")), db.Books.authors.any(db.Authors.name.ilike("%" + term + "%")), @@ -1598,6 +1600,7 @@ def search(): @login_required_if_no_ano def advanced_search(): if request.method == 'GET': + db.session.connection().connection.connection.create_function("lower", 1, db.lcase) q = db.session.query(db.Books) include_tag_inputs = request.args.getlist('include_tag') exclude_tag_inputs = request.args.getlist('exclude_tag') From 3d251bc0b4acf352fb5a693b96a012824c1688d3 Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 21 Oct 2017 21:54:15 +0200 Subject: [PATCH 3/4] accidentally committed changed optional-requirements.txt -> changing back --- optional-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optional-requirements.txt b/optional-requirements.txt index 7a2ad451..cf743dbb 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -12,4 +12,4 @@ rsa==3.4.2 six==1.10.0 uritemplate==3.0.0 goodreads>=0.3.2 -#python-Levenshtein>=0.12.0 +python-Levenshtein>=0.12.0 From 674baad49e18b2a43e113e13f8d1097eb53f5c7a Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 21 Oct 2017 22:07:39 +0200 Subject: [PATCH 4/4] lowercasing the input --- cps/web.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cps/web.py b/cps/web.py index feccd092..3cecfcbb 100755 --- a/cps/web.py +++ b/cps/web.py @@ -659,6 +659,7 @@ def feed_normal_search(): def feed_search(term): if term: + term = term.strip().lower() db.session.connection().connection.connection.create_function("lower", 1, db.lcase) entries = db.session.query(db.Books).filter(db.or_(db.Books.tags.any(db.Tags.name.ilike("%" + term + "%")), db.Books.series.any(db.Series.name.ilike("%" + term + "%")), @@ -1612,9 +1613,9 @@ def advanced_search(): author_name = request.args.get("author_name") book_title = request.args.get("book_title") publisher = request.args.get("publisher") - if author_name: author_name = author_name.strip() - if book_title: book_title = book_title.strip() - if publisher: publisher = publisher.strip() + if author_name: author_name = author_name.strip().lower() + if book_title: book_title = book_title.strip().lower() + if publisher: publisher = publisher.strip().lower() if include_tag_inputs or exclude_tag_inputs or include_series_inputs or exclude_series_inputs or \ include_languages_inputs or exclude_languages_inputs or author_name or book_title or publisher: searchterm = []