diff --git a/.github/workflows/far-build.yml b/.github/workflows/far-build.yml index 9e0d81977b..f8f95c36d1 100644 --- a/.github/workflows/far-build.yml +++ b/.github/workflows/far-build.yml @@ -13,26 +13,38 @@ jobs: strategy: fail-fast: false matrix: - build: [ MSVC_PROJ_x64_Debug,MSVC_PROJ_x64_Release,MSVC_PROJ_x86_Debug,MSVC_PROJ_x86_Release, - MSVC_NMAKE_x64_Debug, MSVC_NMAKE_x64_Release,MSVC_NMAKE_x86_Debug,MSVC_NMAKE_x86_Release, - CLANG_NMAKE_x64_Debug,CLANG_NMAKE_x64_Release, - GCC_MAKE_x64_Debug,GCC_MAKE_x64_Release, - CLANG_MAKE_x64_Debug,CLANG_MAKE_x64_Release ] + build: [ + MSVC_PROJ_x64_Debug, + MSVC_PROJ_x64_Release, + MSVC_PROJ_x86_Debug, + MSVC_PROJ_x86_Release, + MSVC_NMAKE_x64_Debug, + MSVC_NMAKE_x64_Release, + MSVC_NMAKE_x86_Debug, + MSVC_NMAKE_x86_Release, + CLANG_NMAKE_x64_Debug, + CLANG_NMAKE_x64_Release, + GCC_MAKE_x64_Debug, + GCC_MAKE_x64_Release, + CLANG_MAKE_x64_Debug, + CLANG_MAKE_x64_Release + ] + include: - - { build: MSVC_PROJ_x64_Debug, compiler: MSVC_PROJ, arch: x64, build_config: Debug } - - { build: MSVC_PROJ_x64_Release, compiler: MSVC_PROJ, arch: x64, build_config: Release } - - { build: MSVC_PROJ_x86_Debug, compiler: MSVC_PROJ, arch: Win32, build_config: Debug } - - { build: MSVC_PROJ_x86_Release, compiler: MSVC_PROJ, arch: Win32, build_config: Release } - - { build: MSVC_NMAKE_x64_Debug, compiler: MSVC_NMAKE, arch: x64, build_config: Debug } - - { build: MSVC_NMAKE_x64_Release, compiler: MSVC_NMAKE, arch: x64, build_config: Release } - - { build: MSVC_NMAKE_x86_Debug, compiler: MSVC_NMAKE, arch: Win32, build_config: Debug } - - { build: MSVC_NMAKE_x86_Release, compiler: MSVC_NMAKE, arch: Win32, build_config: Release } - - { build: CLANG_NMAKE_x64_Debug, compiler: CLANG_NMAKE, arch: x64, build_config: Debug } - - { build: CLANG_NMAKE_x64_Release, compiler: CLANG_NMAKE, arch: x64, build_config: Release } - - { build: GCC_MAKE_x64_Debug, compiler: GCC_MAKE, arch: x64, build_config: Debug } - - { build: GCC_MAKE_x64_Release, compiler: GCC_MAKE, arch: x64, build_config: Release } - - { build: CLANG_MAKE_x64_Debug, compiler: CLANG_MAKE, arch: x64, build_config: Debug } - - { build: CLANG_MAKE_x64_Release, compiler: CLANG_MAKE, arch: x64, build_config: Release } + - { build: MSVC_PROJ_x64_Debug, compiler: MSVC_PROJ, arch: x64, build_config: Debug } + - { build: MSVC_PROJ_x64_Release, compiler: MSVC_PROJ, arch: x64, build_config: Release } + - { build: MSVC_PROJ_x86_Debug, compiler: MSVC_PROJ, arch: Win32, build_config: Debug } + - { build: MSVC_PROJ_x86_Release, compiler: MSVC_PROJ, arch: Win32, build_config: Release } + - { build: MSVC_NMAKE_x64_Debug, compiler: MSVC_NMAKE, arch: x64, build_config: Debug } + - { build: MSVC_NMAKE_x64_Release, compiler: MSVC_NMAKE, arch: x64, build_config: Release } + - { build: MSVC_NMAKE_x86_Debug, compiler: MSVC_NMAKE, arch: Win32, build_config: Debug } + - { build: MSVC_NMAKE_x86_Release, compiler: MSVC_NMAKE, arch: Win32, build_config: Release } + - { build: CLANG_NMAKE_x64_Debug, compiler: CLANG_NMAKE, arch: x64, build_config: Debug } + - { build: CLANG_NMAKE_x64_Release, compiler: CLANG_NMAKE, arch: x64, build_config: Release } + - { build: GCC_MAKE_x64_Debug, compiler: GCC_MAKE, arch: x64, build_config: Debug } + - { build: GCC_MAKE_x64_Release, compiler: GCC_MAKE, arch: x64, build_config: Release } + - { build: CLANG_MAKE_x64_Debug, compiler: CLANG_MAKE, arch: x64, build_config: Debug } + - { build: CLANG_MAKE_x64_Release, compiler: CLANG_MAKE, arch: x64, build_config: Release } steps: - name: Checkout source @@ -44,9 +56,9 @@ jobs: with: arch: ${{ matrix.arch }} - - name: Set envirement for debug build + - name: Set environment for debug build if: matrix.build_config == 'Debug' - run: set DEBUG=1 + run: echo "DEBUG=1" >> $GITHUB_ENV - name: Build MsBuild if: matrix.compiler == 'MSVC_PROJ' diff --git a/far/changelog b/far/changelog index b0f8d23e62..6a231d497b 100644 --- a/far/changelog +++ b/far/changelog @@ -1,3 +1,8 @@ +-------------------------------------------------------------------------------- +drkns 11.05.2021 02:02:02 +0100 - build 5798 + +1. Continue the "SQLite vs nocase" saga. Now any related errors should be fixed automatically. + -------------------------------------------------------------------------------- drkns 09.05.2021 20:00:49 +0100 - build 5797 diff --git a/far/common/placement.hpp b/far/common/placement.hpp index 3c965b49ed..9b6d5b81cc 100644 --- a/far/common/placement.hpp +++ b/far/common/placement.hpp @@ -51,7 +51,7 @@ namespace placement #ifdef _DEBUG // To increase the chance of crash on use-after-delete - std::memset(&Object, 0xFE, sizeof(Object)); + std::memset(static_cast(&Object), 0xFE, sizeof(Object)); #endif } } diff --git a/far/configdb.cpp b/far/configdb.cpp index 724b258e4a..c6ced7dabf 100644 --- a/far/configdb.cpp +++ b/far/configdb.cpp @@ -533,6 +533,8 @@ class HierarchicalConfigDb: public async_delete_impl, public HierarchicalConfig, { Db.EnableForeignKeysConstraints(); + Db.add_numeric_collation(); + static const std::string_view Schema[] { "CREATE TABLE IF NOT EXISTS table_keys(id INTEGER PRIMARY KEY, parent_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT, FOREIGN KEY(parent_id) REFERENCES table_keys(id) ON UPDATE CASCADE ON DELETE CASCADE, UNIQUE (parent_id,name));"sv, @@ -1207,7 +1209,7 @@ class AssociationsConfigDb: public AssociationsConfig, public sqlite_boilerplate return; SCOPED_ACTION(auto)(ScopedTransaction()); - Exec({ "DELETE FROM filetypes;"sv }); //delete all before importing + Exec("DELETE FROM filetypes;"sv); // delete all before importing unsigned long long id = 0; for (const auto& e: xml_enum(base, "filetype")) { @@ -1273,7 +1275,7 @@ class PluginsCacheConfigDb: public PluginsCacheConfig, public sqlite_boilerplate void DiscardCache() override { SCOPED_ACTION(auto)(ScopedTransaction()); - Exec({ "DELETE FROM cachename;"sv }); + Exec("DELETE FROM cachename;"sv); } private: @@ -1882,10 +1884,16 @@ class HistoryConfigCustom: public HistoryConfig, public sqlite_boilerplate AsyncWork.set(); } + #define EDITORPOSITION_HISTORY_NAME "editorposition_history" + #define VIEWERPOSITION_HISTORY_NAME "viewerposition_history" + #define EDITORPOSITION_HISTORY_SCHEMA "(id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE COLLATE NOCASE, time INTEGER NOT NULL, line INTEGER NOT NULL, linepos INTEGER NOT NULL, screenline INTEGER NOT NULL, leftpos INTEGER NOT NULL, codepage INTEGER NOT NULL);" + #define VIEWERPOSITION_HISTORY_SCHEMA "(id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE COLLATE NOCASE, time INTEGER NOT NULL, filepos INTEGER NOT NULL, leftpos INTEGER NOT NULL, hex INTEGER NOT NULL, codepage INTEGER NOT NULL);" + static void Initialise(const db_initialiser& Db) { Db.SetWALJournalingMode(); - Db.EnableForeignKeysConstraints(); + + Db.add_nocase_collation(); static const std::string_view Schema[] { @@ -1896,16 +1904,21 @@ class HistoryConfigCustom: public HistoryConfig, public sqlite_boilerplate "CREATE INDEX IF NOT EXISTS history_idx3 ON history (kind, key, lock DESC, time DESC);"sv, "CREATE INDEX IF NOT EXISTS history_idx4 ON history (kind, key, time DESC);"sv, //view,edit file positions and bookmarks history - "CREATE TABLE IF NOT EXISTS editorposition_history(id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE COLLATE NOCASE, time INTEGER NOT NULL, line INTEGER NOT NULL, linepos INTEGER NOT NULL, screenline INTEGER NOT NULL, leftpos INTEGER NOT NULL, codepage INTEGER NOT NULL);"sv, + "CREATE TABLE IF NOT EXISTS " EDITORPOSITION_HISTORY_NAME EDITORPOSITION_HISTORY_SCHEMA ""sv, "CREATE TABLE IF NOT EXISTS editorbookmarks_history(pid INTEGER NOT NULL, num INTEGER NOT NULL, line INTEGER NOT NULL, linepos INTEGER NOT NULL, screenline INTEGER NOT NULL, leftpos INTEGER NOT NULL, FOREIGN KEY(pid) REFERENCES editorposition_history(id) ON UPDATE CASCADE ON DELETE CASCADE, PRIMARY KEY (pid, num));"sv, "CREATE INDEX IF NOT EXISTS editorposition_history_idx1 ON editorposition_history (time DESC);"sv, - "CREATE TABLE IF NOT EXISTS viewerposition_history(id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE COLLATE NOCASE, time INTEGER NOT NULL, filepos INTEGER NOT NULL, leftpos INTEGER NOT NULL, hex INTEGER NOT NULL, codepage INTEGER NOT NULL);"sv, + "CREATE TABLE IF NOT EXISTS " VIEWERPOSITION_HISTORY_NAME VIEWERPOSITION_HISTORY_SCHEMA ""sv, "CREATE TABLE IF NOT EXISTS viewerbookmarks_history(pid INTEGER NOT NULL, num INTEGER NOT NULL, filepos INTEGER NOT NULL, leftpos INTEGER NOT NULL, FOREIGN KEY(pid) REFERENCES viewerposition_history(id) ON UPDATE CASCADE ON DELETE CASCADE, PRIMARY KEY (pid, num));"sv, "CREATE INDEX IF NOT EXISTS viewerposition_history_idx1 ON viewerposition_history (time DESC);"sv, }; Db.Exec(Schema); + reindex(Db); + + // Must be after reindex + Db.EnableForeignKeysConstraints(); + static const stmt_init Statements[] { { stmtEnum, "SELECT id, name, type, lock, time, guid, file, data FROM history WHERE kind=?1 AND key=?2 ORDER BY time;"sv }, @@ -1938,6 +1951,54 @@ class HistoryConfigCustom: public HistoryConfig, public sqlite_boilerplate Db.PrepareStatements(Statements); } + static void reindex(db_initialiser const& Db) + { + static const std::pair ReindexTables[] + { + { EDITORPOSITION_HISTORY_NAME ""sv, EDITORPOSITION_HISTORY_SCHEMA ""sv }, + { VIEWERPOSITION_HISTORY_NAME ""sv, VIEWERPOSITION_HISTORY_SCHEMA ""sv }, + }; + + for (const auto& [Name, Schema]: ReindexTables) + { + const auto reindex = [&, Name = Name]{ Db.Exec(format(FSTR("REINDEX {}"sv), Name)); }; + + try + { + reindex(); + } + catch (far_sqlite_exception const& e) + { + if (!e.is_constaint_unique()) + throw; + + recreate_position_history(Db, Name, Schema); + reindex(); + } + } + } + + static void recreate_position_history(const db_initialiser& Db, std::string_view const Table, std::string_view const Schema) + { + LOGNOTICE(L"Recreating {}"sv, encoding::utf8::get_chars(Table)); + + SCOPED_ACTION(auto)(Db.ScopedTransaction()); + + // The order is important - https://sqlite.org/lang_altertable.html + + // 1. Create new table + Db.Exec(format(FSTR("CREATE TABLE {}_new{}"sv), Table, Schema)); + + // 2. Copy data. "WHERE 1=1" is a dirty hack to prevent xfer optimization. + Db.Exec(format(FSTR("INSERT OR IGNORE INTO {0}_new SELECT * FROM {0} WHERE 1=1"sv), Table)); + + // 3. Drop old table + Db.Exec(format(FSTR("DROP TABLE {}"sv), Table)); + + // 4. Rename new into old + Db.Exec(format(FSTR("ALTER TABLE {0}_new RENAME TO {0}"sv), Table)); + } + void Delete(unsigned long long id) override { WaitAllAsync(); diff --git a/far/sqlitedb.cpp b/far/sqlitedb.cpp index 6a280b03c6..b9d51f9778 100644 --- a/far/sqlitedb.cpp +++ b/far/sqlitedb.cpp @@ -118,7 +118,7 @@ namespace [[noreturn]] void throw_exception(string_view const DatabaseName, int const ErrorCode, string_view const ErrorString = {}, int const SystemErrorCode = 0, string_view const Sql = {}) { - throw MAKE_EXCEPTION(far_sqlite_exception, true, format(FSTR(L"[{}] - SQLite error {}: {}{}{}"sv), + throw MAKE_EXCEPTION(far_sqlite_exception, ErrorCode, true, format(FSTR(L"[{}] - SQLite error {}: {}{}{}"sv), DatabaseName, ErrorCode, ErrorString.empty()? GetErrorString(ErrorCode) : ErrorString, @@ -139,14 +139,6 @@ namespace if (!Callable()) { - if (GetLastErrorCode(Db) == SQLITE_CORRUPT_INDEX) - { - // We replace "nocase" collation, so reindexing is required. - // It's quite expensive to do on every start, so do it here when needed. - if (sqlite::sqlite3_exec(Db, "REINDEX NOCASE", {}, {}, {}) == SQLITE_OK && Callable()) - return; - } - throw_exception(Db, {}, SqlAccessor? SqlAccessor() : L""sv); } } @@ -172,13 +164,22 @@ static void sqlite_log(void*, int const Code, const char* const Message) LOG(Level, L"SQLite {} ({}): {}"sv, GetErrorString(Code), Code, encoding::utf8::get_chars(Message)); } +bool far_sqlite_exception::is_constaint_unique() const +{ + return m_ErrorCode == SQLITE_CONSTRAINT_UNIQUE; +} + +bool far_sqlite_exception::is_corrupt_index() const +{ + return m_ErrorCode == SQLITE_CORRUPT_INDEX; +} + void SQLiteDb::library_load() { sqlite::sqlite3_config(SQLITE_CONFIG_LOG, sqlite_log, nullptr); if (const auto Result = sqlite::sqlite3_initialize(); Result != SQLITE_OK) { - assert(false); LOGERROR(L"sqlite3_initialize(): {}"sv, GetErrorString(Result)); } } @@ -187,7 +188,6 @@ void SQLiteDb::library_free() { if (const auto Result = sqlite::sqlite3_shutdown(); Result != SQLITE_OK) { - assert(false); LOGERROR(L"sqlite3_shutdown(): {}"sv, GetErrorString(Result)); } } @@ -206,8 +206,7 @@ void SQLiteDb::SQLiteStmt::stmt_deleter::operator()(sqlite::sqlite3_stmt* Object { if (const auto Result = sqlite::sqlite3_finalize(Object); Result != SQLITE_OK) { - assert(false); - LOGERROR(L"sqlite3_finalize(): {}"sv, GetErrorString(Result)); + LOGDEBUG(L"sqlite3_finalize(): {}"sv, GetErrorString(Result)); } return true; }); @@ -229,8 +228,7 @@ SQLiteDb::SQLiteStmt& SQLiteDb::SQLiteStmt::Reset() { if (const auto Result = sqlite::sqlite3_reset(m_Stmt.get()); Result != SQLITE_OK) { - assert(false); - LOGERROR(L"sqlite3_reset(): {}"sv, GetErrorString(Result)); + LOGDEBUG(L"sqlite3_reset(): {}"sv, GetErrorString(Result)); } return true; }); @@ -386,8 +384,7 @@ void SQLiteDb::db_closer::operator()(sqlite::sqlite3* Object) const noexcept { if (const auto Result = sqlite::sqlite3_close(Object); Result != SQLITE_OK) { - assert(false); - LOGERROR(L"sqlite3_close(): {}"sv, GetLastErrorString(Object)); + LOGDEBUG(L"sqlite3_close(): {}"sv, GetLastErrorString(Object)); } } @@ -454,7 +451,6 @@ class SQLiteDb::implementation { if (const auto Result = sqlite::sqlite3_backup_finish(Backup); Result != SQLITE_OK) { - assert(false); LOGERROR(L"sqlite3_backup_finish(): {}"sv, GetErrorString(Result)); } } @@ -502,6 +498,16 @@ SQLiteDb::database_ptr SQLiteDb::Open(string_view const Path, busy_handler BusyH implementation::open(memory_db_name, {}); } +void SQLiteDb::Exec(std::string const& Command) const +{ + Exec(std::string_view(Command)); +} + +void SQLiteDb::Exec(std::string_view const Command) const +{ + Exec(span{ Command }); +} + void SQLiteDb::Exec(span const Commands) const { for (const auto& i: Commands) @@ -557,12 +563,17 @@ void SQLiteDb::Close() void SQLiteDb::SetWALJournalingMode() const { - Exec({ "PRAGMA journal_mode = WAL;"sv }); + Exec("PRAGMA journal_mode = WAL;"sv); } void SQLiteDb::EnableForeignKeysConstraints() const { - Exec({ "PRAGMA foreign_keys = ON;"sv }); + Exec("PRAGMA foreign_keys = ON;"sv); +} + +void SQLiteDb::initialise() const +{ + sqlite::sqlite3_extended_result_codes(m_Db.get(), true); } template @@ -589,26 +600,28 @@ static int combined_comparer(void* const Param, int const Size1, const void* con ); } -void SQLiteDb::initialise() const -{ - sqlite::sqlite3_extended_result_codes(m_Db.get(), true); - - using comparer_type = int(void*, int, const void*, int, const void*); +using comparer_type = int(void*, int, const void*, int, const void*); - const auto create_collation = [&](const char* const Name, int const Encoding, comparer_type Comparer) +static void create_combined_collation(sqlite::sqlite3* const Db, const char* const Name, comparer_type Comparer) +{ + const auto create_collation = [&](int const Encoding) { - invoke(m_Db.get(), [&] + invoke(Db, [&] { - return sqlite::sqlite3_create_collation(m_Db.get(), Name, Encoding, ToPtr(Encoding), Comparer) == SQLITE_OK; + return sqlite::sqlite3_create_collation(Db, Name, Encoding, ToPtr(Encoding), Comparer) == SQLITE_OK; }); }; - const auto create_combined_collation = [&](const char* const Name, comparer_type Comparer) - { - create_collation(Name, SQLITE_UTF8, Comparer); - create_collation(Name, SQLITE_UTF16, Comparer); - }; + create_collation(SQLITE_UTF8); + create_collation(SQLITE_UTF16); +} - create_combined_collation("nocase", combined_comparer); - create_combined_collation("numeric", combined_comparer); +void SQLiteDb::add_nocase_collation() const +{ + create_combined_collation(m_Db.get(), "nocase", combined_comparer); +} + +void SQLiteDb::add_numeric_collation() const +{ + create_combined_collation(m_Db.get(), "numeric", combined_comparer); } diff --git a/far/sqlitedb.hpp b/far/sqlitedb.hpp index 6bc1d8132d..82b16696bb 100644 --- a/far/sqlitedb.hpp +++ b/far/sqlitedb.hpp @@ -54,9 +54,20 @@ namespace sqlite struct sqlite3_stmt; } -class far_sqlite_exception : public far_exception +class far_sqlite_exception: public far_exception { - using far_exception::far_exception; +public: + template + explicit far_sqlite_exception(int ErrorCode, args&&... Args) : + far_exception(FWD(Args)...), + m_ErrorCode(ErrorCode) + {} + + bool is_constaint_unique() const; + bool is_corrupt_index() const; + +private: + int m_ErrorCode; }; class SQLiteDb: noncopyable, virtual protected transactional @@ -157,6 +168,8 @@ class SQLiteDb: noncopyable, virtual protected transactional }); } + void Exec(std::string const& Command) const; + void Exec(std::string_view Command) const; void Exec(span Commands) const; void SetWALJournalingMode() const; void EnableForeignKeysConstraints() const; @@ -192,6 +205,9 @@ class SQLiteDb: noncopyable, virtual protected transactional FORWARD_FUNCTION(SetWALJournalingMode) FORWARD_FUNCTION(EnableForeignKeysConstraints) FORWARD_FUNCTION(PrepareStatements) + FORWARD_FUNCTION(add_nocase_collation) + FORWARD_FUNCTION(add_numeric_collation) + FORWARD_FUNCTION(ScopedTransaction) #undef FORWARD_FUNCTION @@ -210,6 +226,8 @@ class SQLiteDb: noncopyable, virtual protected transactional void Close(); void initialise() const; + void add_nocase_collation() const; + void add_numeric_collation() const; // The order is important bool m_DbExists{}; diff --git a/far/vbuild.m4 b/far/vbuild.m4 index b4826db971..ac99f6ce97 100644 --- a/far/vbuild.m4 +++ b/far/vbuild.m4 @@ -1 +1 @@ -5797 +5798