Skip to content

Commit

Permalink
Continue the "SQLite vs nocase" saga
Browse files Browse the repository at this point in the history 
Now any related errors should be fixed automatically
  • Loading branch information
@alabuzhev
alabuzhev committed May 11, 2021
1 parent 0235bf1 commit 463f184
Show file tree
Hide file tree
Showing 7 changed files with 174 additions and 65 deletions.
54 changes: 33 additions & 21 deletions .github/workflows/far-build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,26 +13,38 @@ jobs:
strategy:
fail-fast: false
matrix:
build: [ MSVC_PROJ_x64_Debug,MSVC_PROJ_x64_Release,MSVC_PROJ_x86_Debug,MSVC_PROJ_x86_Release,
MSVC_NMAKE_x64_Debug, MSVC_NMAKE_x64_Release,MSVC_NMAKE_x86_Debug,MSVC_NMAKE_x86_Release,
CLANG_NMAKE_x64_Debug,CLANG_NMAKE_x64_Release,
GCC_MAKE_x64_Debug,GCC_MAKE_x64_Release,
CLANG_MAKE_x64_Debug,CLANG_MAKE_x64_Release ]
build: [
MSVC_PROJ_x64_Debug,
MSVC_PROJ_x64_Release,
MSVC_PROJ_x86_Debug,
MSVC_PROJ_x86_Release,
MSVC_NMAKE_x64_Debug,
MSVC_NMAKE_x64_Release,
MSVC_NMAKE_x86_Debug,
MSVC_NMAKE_x86_Release,
CLANG_NMAKE_x64_Debug,
CLANG_NMAKE_x64_Release,
GCC_MAKE_x64_Debug,
GCC_MAKE_x64_Release,
CLANG_MAKE_x64_Debug,
CLANG_MAKE_x64_Release
]

include:
- { build: MSVC_PROJ_x64_Debug, compiler: MSVC_PROJ, arch: x64, build_config: Debug }
- { build: MSVC_PROJ_x64_Release, compiler: MSVC_PROJ, arch: x64, build_config: Release }
- { build: MSVC_PROJ_x86_Debug, compiler: MSVC_PROJ, arch: Win32, build_config: Debug }
- { build: MSVC_PROJ_x86_Release, compiler: MSVC_PROJ, arch: Win32, build_config: Release }
- { build: MSVC_NMAKE_x64_Debug, compiler: MSVC_NMAKE, arch: x64, build_config: Debug }
- { build: MSVC_NMAKE_x64_Release, compiler: MSVC_NMAKE, arch: x64, build_config: Release }
- { build: MSVC_NMAKE_x86_Debug, compiler: MSVC_NMAKE, arch: Win32, build_config: Debug }
- { build: MSVC_NMAKE_x86_Release, compiler: MSVC_NMAKE, arch: Win32, build_config: Release }
- { build: CLANG_NMAKE_x64_Debug, compiler: CLANG_NMAKE, arch: x64, build_config: Debug }
- { build: CLANG_NMAKE_x64_Release, compiler: CLANG_NMAKE, arch: x64, build_config: Release }
- { build: GCC_MAKE_x64_Debug, compiler: GCC_MAKE, arch: x64, build_config: Debug }
- { build: GCC_MAKE_x64_Release, compiler: GCC_MAKE, arch: x64, build_config: Release }
- { build: CLANG_MAKE_x64_Debug, compiler: CLANG_MAKE, arch: x64, build_config: Debug }
- { build: CLANG_MAKE_x64_Release, compiler: CLANG_MAKE, arch: x64, build_config: Release }
- { build: MSVC_PROJ_x64_Debug, compiler: MSVC_PROJ, arch: x64, build_config: Debug }
- { build: MSVC_PROJ_x64_Release, compiler: MSVC_PROJ, arch: x64, build_config: Release }
- { build: MSVC_PROJ_x86_Debug, compiler: MSVC_PROJ, arch: Win32, build_config: Debug }
- { build: MSVC_PROJ_x86_Release, compiler: MSVC_PROJ, arch: Win32, build_config: Release }
- { build: MSVC_NMAKE_x64_Debug, compiler: MSVC_NMAKE, arch: x64, build_config: Debug }
- { build: MSVC_NMAKE_x64_Release, compiler: MSVC_NMAKE, arch: x64, build_config: Release }
- { build: MSVC_NMAKE_x86_Debug, compiler: MSVC_NMAKE, arch: Win32, build_config: Debug }
- { build: MSVC_NMAKE_x86_Release, compiler: MSVC_NMAKE, arch: Win32, build_config: Release }
- { build: CLANG_NMAKE_x64_Debug, compiler: CLANG_NMAKE, arch: x64, build_config: Debug }
- { build: CLANG_NMAKE_x64_Release, compiler: CLANG_NMAKE, arch: x64, build_config: Release }
- { build: GCC_MAKE_x64_Debug, compiler: GCC_MAKE, arch: x64, build_config: Debug }
- { build: GCC_MAKE_x64_Release, compiler: GCC_MAKE, arch: x64, build_config: Release }
- { build: CLANG_MAKE_x64_Debug, compiler: CLANG_MAKE, arch: x64, build_config: Debug }
- { build: CLANG_MAKE_x64_Release, compiler: CLANG_MAKE, arch: x64, build_config: Release }

steps:
- name: Checkout source
Expand All @@ -44,9 +56,9 @@ jobs:
with:
arch: ${{ matrix.arch }}

- name: Set envirement for debug build
- name: Set environment for debug build
if: matrix.build_config == 'Debug'
run: set DEBUG=1
run: echo "DEBUG=1" >> $GITHUB_ENV

- name: Build MsBuild
if: matrix.compiler == 'MSVC_PROJ'
Expand Down
5 changes: 5 additions & 0 deletions far/changelog
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
--------------------------------------------------------------------------------
drkns 11.05.2021 02:02:02 +0100 - build 5798

1. Continue the "SQLite vs nocase" saga. Now any related errors should be fixed automatically.

--------------------------------------------------------------------------------
drkns 09.05.2021 20:00:49 +0100 - build 5797

Expand Down
2 changes: 1 addition & 1 deletion far/common/placement.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ namespace placement

#ifdef _DEBUG
// To increase the chance of crash on use-after-delete
std::memset(&Object, 0xFE, sizeof(Object));
std::memset(static_cast<void*>(&Object), 0xFE, sizeof(Object));
#endif
}
}
Expand Down
71 changes: 66 additions & 5 deletions far/configdb.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -533,6 +533,8 @@ class HierarchicalConfigDb: public async_delete_impl, public HierarchicalConfig,
{
Db.EnableForeignKeysConstraints();

Db.add_numeric_collation();

static const std::string_view Schema[]
{
"CREATE TABLE IF NOT EXISTS table_keys(id INTEGER PRIMARY KEY, parent_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT, FOREIGN KEY(parent_id) REFERENCES table_keys(id) ON UPDATE CASCADE ON DELETE CASCADE, UNIQUE (parent_id,name));"sv,
Expand Down Expand Up @@ -1207,7 +1209,7 @@ class AssociationsConfigDb: public AssociationsConfig, public sqlite_boilerplate
return;

SCOPED_ACTION(auto)(ScopedTransaction());
Exec({ "DELETE FROM filetypes;"sv }); //delete all before importing
Exec("DELETE FROM filetypes;"sv); // delete all before importing
unsigned long long id = 0;
for (const auto& e: xml_enum(base, "filetype"))
{
Expand Down Expand Up @@ -1273,7 +1275,7 @@ class PluginsCacheConfigDb: public PluginsCacheConfig, public sqlite_boilerplate
void DiscardCache() override
{
SCOPED_ACTION(auto)(ScopedTransaction());
Exec({ "DELETE FROM cachename;"sv });
Exec("DELETE FROM cachename;"sv);
}

private:
Expand Down Expand Up @@ -1882,10 +1884,16 @@ class HistoryConfigCustom: public HistoryConfig, public sqlite_boilerplate
AsyncWork.set();
}

#define EDITORPOSITION_HISTORY_NAME "editorposition_history"
#define VIEWERPOSITION_HISTORY_NAME "viewerposition_history"
#define EDITORPOSITION_HISTORY_SCHEMA "(id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE COLLATE NOCASE, time INTEGER NOT NULL, line INTEGER NOT NULL, linepos INTEGER NOT NULL, screenline INTEGER NOT NULL, leftpos INTEGER NOT NULL, codepage INTEGER NOT NULL);"
#define VIEWERPOSITION_HISTORY_SCHEMA "(id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE COLLATE NOCASE, time INTEGER NOT NULL, filepos INTEGER NOT NULL, leftpos INTEGER NOT NULL, hex INTEGER NOT NULL, codepage INTEGER NOT NULL);"

static void Initialise(const db_initialiser& Db)
{
Db.SetWALJournalingMode();
Db.EnableForeignKeysConstraints();

Db.add_nocase_collation();

static const std::string_view Schema[]
{
Expand All @@ -1896,16 +1904,21 @@ class HistoryConfigCustom: public HistoryConfig, public sqlite_boilerplate
"CREATE INDEX IF NOT EXISTS history_idx3 ON history (kind, key, lock DESC, time DESC);"sv,
"CREATE INDEX IF NOT EXISTS history_idx4 ON history (kind, key, time DESC);"sv,
//view,edit file positions and bookmarks history
"CREATE TABLE IF NOT EXISTS editorposition_history(id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE COLLATE NOCASE, time INTEGER NOT NULL, line INTEGER NOT NULL, linepos INTEGER NOT NULL, screenline INTEGER NOT NULL, leftpos INTEGER NOT NULL, codepage INTEGER NOT NULL);"sv,
"CREATE TABLE IF NOT EXISTS " EDITORPOSITION_HISTORY_NAME EDITORPOSITION_HISTORY_SCHEMA ""sv,
"CREATE TABLE IF NOT EXISTS editorbookmarks_history(pid INTEGER NOT NULL, num INTEGER NOT NULL, line INTEGER NOT NULL, linepos INTEGER NOT NULL, screenline INTEGER NOT NULL, leftpos INTEGER NOT NULL, FOREIGN KEY(pid) REFERENCES editorposition_history(id) ON UPDATE CASCADE ON DELETE CASCADE, PRIMARY KEY (pid, num));"sv,
"CREATE INDEX IF NOT EXISTS editorposition_history_idx1 ON editorposition_history (time DESC);"sv,
"CREATE TABLE IF NOT EXISTS viewerposition_history(id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE COLLATE NOCASE, time INTEGER NOT NULL, filepos INTEGER NOT NULL, leftpos INTEGER NOT NULL, hex INTEGER NOT NULL, codepage INTEGER NOT NULL);"sv,
"CREATE TABLE IF NOT EXISTS " VIEWERPOSITION_HISTORY_NAME VIEWERPOSITION_HISTORY_SCHEMA ""sv,
"CREATE TABLE IF NOT EXISTS viewerbookmarks_history(pid INTEGER NOT NULL, num INTEGER NOT NULL, filepos INTEGER NOT NULL, leftpos INTEGER NOT NULL, FOREIGN KEY(pid) REFERENCES viewerposition_history(id) ON UPDATE CASCADE ON DELETE CASCADE, PRIMARY KEY (pid, num));"sv,
"CREATE INDEX IF NOT EXISTS viewerposition_history_idx1 ON viewerposition_history (time DESC);"sv,
};

Db.Exec(Schema);

reindex(Db);

// Must be after reindex
Db.EnableForeignKeysConstraints();

static const stmt_init<statement_id> Statements[]
{
{ stmtEnum, "SELECT id, name, type, lock, time, guid, file, data FROM history WHERE kind=?1 AND key=?2 ORDER BY time;"sv },
Expand Down Expand Up @@ -1938,6 +1951,54 @@ class HistoryConfigCustom: public HistoryConfig, public sqlite_boilerplate
Db.PrepareStatements(Statements);
}

static void reindex(db_initialiser const& Db)
{
static const std::pair<std::string_view, std::string_view> ReindexTables[]
{
{ EDITORPOSITION_HISTORY_NAME ""sv, EDITORPOSITION_HISTORY_SCHEMA ""sv },
{ VIEWERPOSITION_HISTORY_NAME ""sv, VIEWERPOSITION_HISTORY_SCHEMA ""sv },
};

for (const auto& [Name, Schema]: ReindexTables)
{
const auto reindex = [&, Name = Name]{ Db.Exec(format(FSTR("REINDEX {}"sv), Name)); };

try
{
reindex();
}
catch (far_sqlite_exception const& e)
{
if (!e.is_constaint_unique())
throw;

recreate_position_history(Db, Name, Schema);
reindex();
}
}
}

static void recreate_position_history(const db_initialiser& Db, std::string_view const Table, std::string_view const Schema)
{
LOGNOTICE(L"Recreating {}"sv, encoding::utf8::get_chars(Table));

SCOPED_ACTION(auto)(Db.ScopedTransaction());

// The order is important - https://sqlite.org/lang_altertable.html

// 1. Create new table
Db.Exec(format(FSTR("CREATE TABLE {}_new{}"sv), Table, Schema));

// 2. Copy data. "WHERE 1=1" is a dirty hack to prevent xfer optimization.
Db.Exec(format(FSTR("INSERT OR IGNORE INTO {0}_new SELECT * FROM {0} WHERE 1=1"sv), Table));

// 3. Drop old table
Db.Exec(format(FSTR("DROP TABLE {}"sv), Table));

// 4. Rename new into old
Db.Exec(format(FSTR("ALTER TABLE {0}_new RENAME TO {0}"sv), Table));
}

void Delete(unsigned long long id) override
{
WaitAllAsync();
Expand Down
83 changes: 48 additions & 35 deletions far/sqlitedb.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ namespace
[[noreturn]]
void throw_exception(string_view const DatabaseName, int const ErrorCode, string_view const ErrorString = {}, int const SystemErrorCode = 0, string_view const Sql = {})
{
throw MAKE_EXCEPTION(far_sqlite_exception, true, format(FSTR(L"[{}] - SQLite error {}: {}{}{}"sv),
throw MAKE_EXCEPTION(far_sqlite_exception, ErrorCode, true, format(FSTR(L"[{}] - SQLite error {}: {}{}{}"sv),
DatabaseName,
ErrorCode,
ErrorString.empty()? GetErrorString(ErrorCode) : ErrorString,
Expand All @@ -139,14 +139,6 @@ namespace

if (!Callable())
{
if (GetLastErrorCode(Db) == SQLITE_CORRUPT_INDEX)
{
// We replace "nocase" collation, so reindexing is required.
// It's quite expensive to do on every start, so do it here when needed.
if (sqlite::sqlite3_exec(Db, "REINDEX NOCASE", {}, {}, {}) == SQLITE_OK && Callable())
return;
}

throw_exception(Db, {}, SqlAccessor? SqlAccessor() : L""sv);
}
}
Expand All @@ -172,13 +164,22 @@ static void sqlite_log(void*, int const Code, const char* const Message)
LOG(Level, L"SQLite {} ({}): {}"sv, GetErrorString(Code), Code, encoding::utf8::get_chars(Message));
}

bool far_sqlite_exception::is_constaint_unique() const
{
return m_ErrorCode == SQLITE_CONSTRAINT_UNIQUE;
}

bool far_sqlite_exception::is_corrupt_index() const
{
return m_ErrorCode == SQLITE_CORRUPT_INDEX;
}

void SQLiteDb::library_load()
{
sqlite::sqlite3_config(SQLITE_CONFIG_LOG, sqlite_log, nullptr);

if (const auto Result = sqlite::sqlite3_initialize(); Result != SQLITE_OK)
{
assert(false);
LOGERROR(L"sqlite3_initialize(): {}"sv, GetErrorString(Result));
}
}
Expand All @@ -187,7 +188,6 @@ void SQLiteDb::library_free()
{
if (const auto Result = sqlite::sqlite3_shutdown(); Result != SQLITE_OK)
{
assert(false);
LOGERROR(L"sqlite3_shutdown(): {}"sv, GetErrorString(Result));
}
}
Expand All @@ -206,8 +206,7 @@ void SQLiteDb::SQLiteStmt::stmt_deleter::operator()(sqlite::sqlite3_stmt* Object
{
if (const auto Result = sqlite::sqlite3_finalize(Object); Result != SQLITE_OK)
{
assert(false);
LOGERROR(L"sqlite3_finalize(): {}"sv, GetErrorString(Result));
LOGDEBUG(L"sqlite3_finalize(): {}"sv, GetErrorString(Result));
}
return true;
});
Expand All @@ -229,8 +228,7 @@ SQLiteDb::SQLiteStmt& SQLiteDb::SQLiteStmt::Reset()
{
if (const auto Result = sqlite::sqlite3_reset(m_Stmt.get()); Result != SQLITE_OK)
{
assert(false);
LOGERROR(L"sqlite3_reset(): {}"sv, GetErrorString(Result));
LOGDEBUG(L"sqlite3_reset(): {}"sv, GetErrorString(Result));
}
return true;
});
Expand Down Expand Up @@ -386,8 +384,7 @@ void SQLiteDb::db_closer::operator()(sqlite::sqlite3* Object) const noexcept
{
if (const auto Result = sqlite::sqlite3_close(Object); Result != SQLITE_OK)
{
assert(false);
LOGERROR(L"sqlite3_close(): {}"sv, GetLastErrorString(Object));
LOGDEBUG(L"sqlite3_close(): {}"sv, GetLastErrorString(Object));
}
}

Expand Down Expand Up @@ -454,7 +451,6 @@ class SQLiteDb::implementation
{
if (const auto Result = sqlite::sqlite3_backup_finish(Backup); Result != SQLITE_OK)
{
assert(false);
LOGERROR(L"sqlite3_backup_finish(): {}"sv, GetErrorString(Result));
}
}
Expand Down Expand Up @@ -502,6 +498,16 @@ SQLiteDb::database_ptr SQLiteDb::Open(string_view const Path, busy_handler BusyH
implementation::open(memory_db_name, {});
}

void SQLiteDb::Exec(std::string const& Command) const
{
Exec(std::string_view(Command));
}

void SQLiteDb::Exec(std::string_view const Command) const
{
Exec(span{ Command });
}

void SQLiteDb::Exec(span<std::string_view const> const Commands) const
{
for (const auto& i: Commands)
Expand Down Expand Up @@ -557,12 +563,17 @@ void SQLiteDb::Close()

void SQLiteDb::SetWALJournalingMode() const
{
Exec({ "PRAGMA journal_mode = WAL;"sv });
Exec("PRAGMA journal_mode = WAL;"sv);
}

void SQLiteDb::EnableForeignKeysConstraints() const
{
Exec({ "PRAGMA foreign_keys = ON;"sv });
Exec("PRAGMA foreign_keys = ON;"sv);
}

void SQLiteDb::initialise() const
{
sqlite::sqlite3_extended_result_codes(m_Db.get(), true);
}

template<typename char_type>
Expand All @@ -589,26 +600,28 @@ static int combined_comparer(void* const Param, int const Size1, const void* con
);
}

void SQLiteDb::initialise() const
{
sqlite::sqlite3_extended_result_codes(m_Db.get(), true);

using comparer_type = int(void*, int, const void*, int, const void*);
using comparer_type = int(void*, int, const void*, int, const void*);

const auto create_collation = [&](const char* const Name, int const Encoding, comparer_type Comparer)
static void create_combined_collation(sqlite::sqlite3* const Db, const char* const Name, comparer_type Comparer)
{
const auto create_collation = [&](int const Encoding)
{
invoke(m_Db.get(), [&]
invoke(Db, [&]
{
return sqlite::sqlite3_create_collation(m_Db.get(), Name, Encoding, ToPtr(Encoding), Comparer) == SQLITE_OK;
return sqlite::sqlite3_create_collation(Db, Name, Encoding, ToPtr(Encoding), Comparer) == SQLITE_OK;
});
};

const auto create_combined_collation = [&](const char* const Name, comparer_type Comparer)
{
create_collation(Name, SQLITE_UTF8, Comparer);
create_collation(Name, SQLITE_UTF16, Comparer);
};
create_collation(SQLITE_UTF8);
create_collation(SQLITE_UTF16);
}

create_combined_collation("nocase", combined_comparer<string_sort::keyhole::compare_ordinal_icase>);
create_combined_collation("numeric", combined_comparer<string_sort::keyhole::compare_ordinal_numeric>);
void SQLiteDb::add_nocase_collation() const
{
create_combined_collation(m_Db.get(), "nocase", combined_comparer<string_sort::keyhole::compare_ordinal_icase>);
}

void SQLiteDb::add_numeric_collation() const
{
create_combined_collation(m_Db.get(), "numeric", combined_comparer<string_sort::keyhole::compare_ordinal_numeric>);
}
Loading

0 comments on commit 463f184

Please sign in to comment.