Repository: SOCI/soci
Pull request #1229: Use built-in SQLite if system library not found
Author: login: vadz
Created: 2025-04-10T16:53:55Z | Merged: 2025-04-11T00:48:10Z

In particular, this fixes AppVeyor CI build as it didn't work with CMake 4.0 any more.


Diff:
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 000000000..4019cba17
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "3rdparty/sqlite3"]
+	path = 3rdparty/sqlite3
+	url = https://github.com/vadz/sqlite-amalgamation.git
diff --git a/3rdparty/sqlite3 b/3rdparty/sqlite3
new file mode 160000
index 000000000..46a41f159
--- /dev/null
+++ b/3rdparty/sqlite3
@@ -0,0 +1 @@
+Subproject commit 46a41f159b163d50fbb914c729d237e6b54ae72d
diff --git a/CMakeLists.txt b/CMakeLists.txt
index d9626be50..83d791e2b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -14,7 +14,7 @@ project(SOCI
   VERSION ${SOCI_VERSION}
   DESCRIPTION "C++ database access library"
   HOMEPAGE_URL "https://soci.sourceforge.net/"
-  LANGUAGES CXX
+  LANGUAGES C CXX
 )
 
 include(soci_utils)
diff --git a/appveyor.yml b/appveyor.yml
index 52c3016ce..1dc87b5c3 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -72,13 +72,12 @@ install:
         $env:SOCI_ODBC_SKIP_TESTS = $env:SOCI_ODBC_SKIP_TESTS + '|soci_odbc_test_postgresql'
       }
       Write-Output "To be skipped ODBC tests: $env:SOCI_ODBC_SKIP_TESTS"
-  - git clone https://github.com/snikulov/sqlite.cmake.build.git C:\projects\sqlite\src
     # Ensure we have a recent cmake version available
   - choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System'
   - refreshenv
+  - git submodule update --init
 
 before_build:
-  - set SQLITE_ROOT=C:\projects\sqlite\sqlite
   # dirty little hack - remove sh from Git to make generator happy
   - ps: |
       if ($env:G -eq "MinGW Makefiles")
@@ -91,12 +90,10 @@ before_build:
             Remove-Item $shellPath
           }
         }
-        $env:SQLITE3_LIBRARY = $env:SQLITE_ROOT + '/lib/libsqlite3-static.a'
         $env:BUILD_TOOL_OPTIONS = '-j3'
       }
       else
       {
-        $env:SQLITE3_LIBRARY = $env:SQLITE_ROOT + '/lib/sqlite3-static.lib'
         $env:BUILD_TOOL_OPTIONS = ''
         # Set up Visual Studio environment (MSVC compiler etc.)
         # Works by calling the respective bat script and then copying over the env vars
@@ -108,9 +105,6 @@ before_build:
                 }
             }
       }
-  - cd C:\projects\sqlite\src
-  - mkdir build
-  - cd build
   - set PATH=%MINGW_BIN%;%POSTGRESQL_ROOT%\bin;%MYSQL_DIR%\bin;%MYSQL_DIR%\lib;%PATH%
   - echo %PATH%
   - cmake --version
@@ -121,14 +115,12 @@ before_build:
   - set USER=root
   - mysql -e "create database soci_test;" --user=root
   - sqlcmd -U sa -P Password12! -S (local)\SQL%MSSQL_VER% -i C:\projects\soci\scripts\windows\mssql_db_create.sql
-  - cmake .. -G"%G%" -DSQLITE_BUILD_SHARED=OFF -DCMAKE_BUILD_TYPE=%CONFIGURATION% -DCMAKE_INSTALL_PREFIX=%SQLITE_ROOT% -DCMAKE_UNITY_BUILD=ON
-  - cmake --build . --config %CONFIGURATION% --target install
 
 build_script:
   - cd C:\projects\soci
   - mkdir build
   - cd build
-  - cmake .. -G"%G%" -DCMAKE_BUILD_TYPE=%CONFIGURATION% -DCMAKE_VERBOSE_MAKEFILE=ON -DSOCI_ENABLE_WERROR=ON -DCMAKE_PREFIX_PATH=%SQLITE_ROOT% -DSQLite3_LIBRARY=%SQLITE3_LIBRARY% -DCMAKE_UNITY_BUILD=ON
+  - cmake .. -G"%G%" -DCMAKE_BUILD_TYPE=%CONFIGURATION% -DCMAKE_VERBOSE_MAKEFILE=ON -DSOCI_ENABLE_WERROR=ON -DCMAKE_UNITY_BUILD=ON
   - cmake --build . --config %CONFIGURATION% -- %BUILD_TOOL_OPTIONS%
 
 test_script:
diff --git a/cmake/soci_define_backend_target.cmake b/cmake/soci_define_backend_target.cmake
index 18b42fef5..eab887938 100644
--- a/cmake/soci_define_backend_target.cmake
+++ b/cmake/soci_define_backend_target.cmake
@@ -9,7 +9,7 @@ include(soci_utils)
 # BACKEND_NAME <name>                      Name of the backend
 # ENABLED_VARIABLE <variable>              CMake variable that indicates whether this backend is enabled. Will be set
 #                                          to OFF if one of the dependencies are not satisfied.
-# MISSING_DEPENDENCY_BEHAVIOR <behavior>   What to do if a dependency is not found. Valid values are "ERROR" and "DISABLE"
+# MISSING_DEPENDENCY_BEHAVIOR <behavior>   What to do if a dependency is not found. Valid values are "ERROR", "DISABLE" and "BUILTIN".
 # TARGET_NAME <target>                     Name of the CMake target that shall be created for this backend
 # DEPENDENCIES <spec1> [... <specN>]       List of dependency specifications. Each specification has to be a single
 #                                          argument (single string) following the syntax
@@ -58,6 +58,9 @@ function(soci_define_backend_target)
     set(ERROR_ON_MISSING_DEPENDENCY ON)
   elseif(DEFINE_BACKEND_MISSING_DEPENDENCY_BEHAVIOR STREQUAL "DISABLE")
     set(DISABLE_ON_MISSING_DEPENDENCY ON)
+  elseif(DEFINE_BACKEND_MISSING_DEPENDENCY_BEHAVIOR STREQUAL "BUILTIN")
+    set(REQUIRE_FLAG "QUIET")
+    set(BUILTIN_ON_MISSING_DEPENDENCY ON)
   else()
     message(FATAL_ERROR "Invalid value '${DEFINE_BACKEND_MISSING_DEPENDENCY_BEHAVIOR}' for option 'MISSING_DEPENDENCY_BEHAVIOR'")
   endif()
@@ -88,22 +91,29 @@ function(soci_define_backend_target)
         get_property(DESCRIPTION CACHE "${DEFINE_BACKEND_ENABLED_VARIABLE}" PROPERTY HELPSTRING)
         set(${DEFINE_BACKEND_ENABLED_VARIABLE} OFF CACHE STRING "${DESCRIPTION}" FORCE)
         return()
+      elseif(BUILTIN_ON_MISSING_DEPENDENCY)
+        message(STATUS "Falling back on built-in version of '${CURRENT_DEP}' for SOCI backend '${DEFINE_BACKEND_BACKEND_NAME}'")
       else()
         message(FATAL_ERROR "Unspecified handling of unmet dependency")
       endif()
-    endif()
+    else()
+      # This is wasteful, but do it again without "QUIET" flag to show the result if it succeeded.
+      if (REQUIRE_FLAG STREQUAL "QUIET")
+        find_package(${CURRENT_DEP_SEARCH})
+      endif()
 
-    foreach (CURRENT IN LISTS CURRENT_DEP_TARGETS)
-      if (NOT TARGET "${CURRENT}")
-        message(FATAL_ERROR "Expected successful find_package call with '${CURRENT_DEP_SEARCH}' to define target '${CURRENT}'")
+      foreach (CURRENT IN LISTS CURRENT_DEP_TARGETS)
+        if (NOT TARGET "${CURRENT}")
+          message(FATAL_ERROR "Expected successful find_package call with '${CURRENT_DEP_SEARCH}' to define target '${CURRENT}'")
+        endif()
+      endforeach()
+      if (CURRENT_DEP_DEFINES)
+        set(MACRO_NAMES_ARG "MACRO_NAMES ${CURRENT_DEP_DEFINES}")
       endif()
-    endforeach()
-    if (CURRENT_DEP_DEFINES)
-      set(MACRO_NAMES_ARG "MACRO_NAMES ${CURRENT_DEP_DEFINES}")
+      list(APPEND PUBLIC_DEP_CALL_ARGS
+        "NAME ${CURRENT_DEP} DEP_TARGETS ${CURRENT_DEP_TARGETS} TARGET SOCI::${DEFINE_BACKEND_ALIAS_NAME} ${MACRO_NAMES_ARG} REQUIRED"
+      )
     endif()
-    list(APPEND PUBLIC_DEP_CALL_ARGS
-      "NAME ${CURRENT_DEP} DEP_TARGETS ${CURRENT_DEP_TARGETS} TARGET SOCI::${DEFINE_BACKEND_ALIAS_NAME} ${MACRO_NAMES_ARG} REQUIRED"
-    )
   endforeach()
 
 
diff --git a/docs/backends/sqlite3.md b/docs/backends/sqlite3.md
index 088f53e18..ed09c974b 100644
--- a/docs/backends/sqlite3.md
+++ b/docs/backends/sqlite3.md
@@ -6,32 +6,11 @@ SOCI backend for accessing SQLite 3 database.
 
 ### Supported Versions
 
-The SOCI SQLite3 backend is supported for use with SQLite3 >= 3.1
-
-### Tested Platforms
-
-|SQLite3|OS|Compiler|
-|--- |--- |--- |
-|3.12.1|Windows Server 2016|MSVC++ 14.1|
-|3.12.1|Windows Server 2012 R2|MSVC++ 14.0|
-|3.12.1|Windows Server 2012 R2|MSVC++ 12.0|
-|3.12.1|Windows Server 2012 R2|MSVC++ 11.0|
-|3.12.1|Windows Server 2012 R2|Mingw-w64/GCC 4.8|
-|3.7.9|Ubuntu 12.04|g++ 4.6.3|
-|3.4.0|Windows XP|(cygwin) g++ 3.4.4|
-|3.4.0|Windows XP|Visual C++ 2005 Express Edition|
-|3.3.8|Windows XP|Visual C++ 2005 Professional|
-|3.5.2|Mac OS X 10.5|g++ 4.0.1|
-|3.3.4|Ubuntu 5.1|g++ 4.0.2|
-|3.3.4|Windows XP|(cygwin) g++ 3.3.4|
-|3.3.4|Windows XP|Visual C++ 2005 Express Edition|
-|3.2.1|Linux i686 2.6.10-gentoo-r6|g++ 3.4.5|
-|3.1.3|Mac OS X 10.4|g++ 4.0.1|
-|3.24.0|macOS High Sierra 10.13.5|AppleClang 9.1.0.9020039|
+The oldest tested SQLite3 version that SOCI was tested with was 3.1.3 and the latest tested version is 3.49.1. Due to high degree of SQLite3 backwards compatibility, it is likely that newer versions can work as well.
 
 ### Required Client Libraries
 
-The SOCI SQLite3 backend requires SQLite3's `libsqlite3` client library.
+The SOCI SQLite3 backend uses `libsqlite3` client library if available or the built-in version of SQLite3 included in the SOCI source code if this library cannot be found.
 
 ### Connecting to the Database
 
@@ -171,6 +150,10 @@ The SQLite3 backend provides the following concrete classes for navite API acces
 
 SQLite3 result code is provided via the backend specific `sqlite3_soci_error` class. Catching the backend specific error yields the value of SQLite3 result code via the `result()` method.
 
+### SQLite3 version information
+
+`sqlie3_session_backend` class, declared in `<soci/sqlite3/soci-sqlite3.h>`, provides static `libversion_number()` and `libversion()` functions which can be used to retrieve the SQLite3 version as a number (e.g. `3049001`) and as a string (e.g. `3.49.1`) respectively.
+
 ## Configuration options
 
 None
diff --git a/docs/installation.md b/docs/installation.md
index 90ef255f5..0770d26e4 100644
--- a/docs/installation.md
+++ b/docs/installation.md
@@ -27,7 +27,7 @@ where X.Y.Z is the version number. Unpack the archive.
 You can always clone SOCI from the Git repository:
 
 ```console
-git clone git://github.com/SOCI/soci.git
+git clone --recurse-submodules git://github.com/SOCI/soci.git
 ```
 
 ## Building with CMake
@@ -145,7 +145,7 @@ Furthermore, the `MYSQL_DIR` _environment variable_ can be set to the MySQL inst
 
 #### SQLite 3
 
-* `SOCI_SQLITE3` - Enabler - Enables the [SQLite3](backends/sqlite3.md) backend.
+* `SOCI_SQLITE3` - Enabler - Enables the [SQLite3](backends/sqlite3.md) backend. Note that, unlike with all the other backends, if SQLite3 library is not found, built-in version of SQLite3 is used instead of the backend being disabled.
 * `SOCI_SQLITE3_TEST_CONNSTR` - string - Connection string is simply a file path where SQLite3 test database will be created (e.g. /home/john/soci_test.db). Check [SQLite3 backend reference](backends/sqlite3.md) for details. Example: `-DSOCI_SQLITE3_TEST_CONNSTR="my.db"` or `-DSOCI_SQLITE3_TEST_CONNSTR=":memory:"`.
 * `SOCI_SQLITE3_SKIP_TESTS` - boolean - Skips testing this backend.
 
diff --git a/include/soci/sqlite3/soci-sqlite3.h b/include/soci/sqlite3/soci-sqlite3.h
index 3bcc85102..407904010 100644
--- a/include/soci/sqlite3/soci-sqlite3.h
+++ b/include/soci/sqlite3/soci-sqlite3.h
@@ -31,19 +31,27 @@
 namespace sqlite_api
 {
 
-#if SQLITE_VERSION_NUMBER < 3003010
-// The sqlite3_destructor_type typedef introduced in 3.3.10
-// https://www.sqlite.org/cvstrac/tktview?tn=2191
-typedef void (*sqlite3_destructor_type)(void*);
+// Don't include sqlite3.h from outside SOCI: this header might not be
+// available when using built-in SQLite3 as we don't install it in this case.
+#ifdef SOCI_SQLITE3_SOURCE
+    #include <sqlite3.h>
+#else // !SOCI_SQLITE3_SOURCE
+// We need just a couple of forward declarations to make this header itself
+// compile.
+struct sqlite3;
+struct sqlite3_stmt;
+
+#if defined(_MSC_VER)
+  typedef __int64 sqlite3_int64;
+  typedef unsigned __int64 sqlite3_uint64;
+#else
+  typedef long long int sqlite3_int64;
+  typedef unsigned long long int sqlite3_uint64;
 #endif
-
-#include <sqlite3.h>
+#endif // SOCI_SQLITE3_SOURCE/!SOCI_SQLITE3_SOURCE
 
 } // namespace sqlite_api
 
-#undef SQLITE_STATIC
-#define SQLITE_STATIC ((sqlite_api::sqlite3_destructor_type)0)
-
 #ifdef _MSC_VER
 #pragma warning(pop)
 #endif
@@ -362,6 +370,10 @@ struct SOCI_SQLITE3_DECL sqlite3_session_backend : details::session_backend
         }
     }
 
+    // Get information about SQLite3 version used.
+    static const char* libversion();
+    static int libversion_number();
+
     sqlite_api::sqlite3 *conn_;
 
     // This flag is set to true if the internal sqlite_sequence table exists in
diff --git a/src/backends/sqlite3/CMakeLists.txt b/src/backends/sqlite3/CMakeLists.txt
index 200a08dbd..9f2db416e 100644
--- a/src/backends/sqlite3/CMakeLists.txt
+++ b/src/backends/sqlite3/CMakeLists.txt
@@ -1,7 +1,9 @@
 include(soci_define_backend_target)
 
 if (SOCI_SQLITE3_AUTO)
-  set(DEPENDENCY_MODE "DISABLE")
+  # Unlike most other backends, we can fall back on using the built-in
+  # version of SQLite3 if the system version is not found.
+  set(DEPENDENCY_MODE "BUILTIN")
 else()
   set(DEPENDENCY_MODE "ERROR")
 endif()
@@ -32,3 +34,31 @@ soci_define_backend_target(
 if (NOT SOCI_SQLITE3)
   return()
 endif()
+
+# If SQLite3 was found, this target must have been created.
+if (NOT TARGET SQLite::SQLite3)
+  # But if it wasn't, use our built-in version, after checking that it is
+  # available.
+  if (NOT EXISTS "${PROJECT_SOURCE_DIR}/3rdparty/sqlite3/sqlite3.c")
+    message(WARNING "SQLite3 not found and built-in version not available, have you cloned SOCI repository with --recurse-submodules?")
+
+    get_property(DESCRIPTION CACHE SOCI_SQLITE3 PROPERTY HELPSTRING)
+    set(SOCI_SQLITE3 OFF CACHE STRING "${DESCRIPTION}" FORCE)
+    return()
+  endif()
+
+  add_library(soci_sqlite3_builtin)
+  target_sources(soci_sqlite3_builtin
+    PRIVATE
+      "${PROJECT_SOURCE_DIR}/3rdparty/sqlite3/sqlite3.c"
+  )
+  target_include_directories(soci_sqlite3_builtin
+    PUBLIC
+      "${PROJECT_SOURCE_DIR}/3rdparty/sqlite3"
+  )
+
+  target_link_libraries(soci_sqlite3
+    PRIVATE
+      soci_sqlite3_builtin
+  )
+endif()
diff --git a/src/backends/sqlite3/session.cpp b/src/backends/sqlite3/session.cpp
index 6b61bbe7b..63cb445eb 100644
--- a/src/backends/sqlite3/session.cpp
+++ b/src/backends/sqlite3/session.cpp
@@ -313,3 +313,15 @@ sqlite3_blob_backend * sqlite3_session_backend::make_blob_backend()
 {
     return new sqlite3_blob_backend(*this);
 }
+
+// static
+const char* sqlite3_session_backend::libversion()
+{
+    return sqlite3_libversion();
+}
+
+// static
+int sqlite3_session_backend::libversion_number()
+{
+    return sqlite3_libversion_number();
+}
diff --git a/tests/sqlite3/test-sqlite3.cpp b/tests/sqlite3/test-sqlite3.cpp
index 143093619..4c5355c26 100644
--- a/tests/sqlite3/test-sqlite3.cpp
+++ b/tests/sqlite3/test-sqlite3.cpp
@@ -404,11 +404,11 @@ TEST_CASE("SQLite long long", "[sqlite][longlong]")
 // Test the DDL and metadata functionality
 TEST_CASE("SQLite DDL with metadata", "[sqlite][ddl]")
 {
-    if (sqlite_api::sqlite3_libversion_number() < 3036000) {
-        if (sqlite_api::sqlite3_libversion_number() < 3014000) {
-            WARN("SQLite requires at least version 3.14.0 for column description, detected " << sqlite_api::sqlite3_libversion());
+    if (sqlite3_session_backend::libversion_number() < 3036000) {
+        if (sqlite3_session_backend::libversion_number() < 3014000) {
+            WARN("SQLite requires at least version 3.14.0 for column description, detected " << sqlite3_session_backend::libversion());
         }
-        WARN("SQLite requires at least version 3.36.0 for drop column, detected " << sqlite_api::sqlite3_libversion());
+        WARN("SQLite requires at least version 3.36.0 for drop column, detected " << sqlite3_session_backend::libversion());
         return;
     }
     soci::session sql(backEnd, connectString);
diff --git a/www/index.html b/www/index.html
index f8e605276..bb9a8e25a 100644
--- a/www/index.html
+++ b/www/index.html
@@ -107,7 +107,7 @@
 
 <p>The main Git repository with SOCI source code can be cloned with:</p>
 <pre>
-$ git clone git://github.com/SOCI/soci.git
+$ git clone --recurse-submodules git://github.com/SOCI/soci.git
 </pre>
 
 <p>The <a href="https://github.com/SOCI/soci/issues">Issues</a> tracker is open


Reviews:
user: login: vadz
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-10T19:58:44Z
user: login: vadz
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-11T00:06:27Z
user: login: Krzmbrzl
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-11T07:20:43Z
user: login: vadz
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-11T13:41:19Z
user: login: Krzmbrzl
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-11T16:21:41Z
user: login: vadz
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-11T16:39:56Z
user: login: Krzmbrzl
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-12T08:51:35Z
user: login: Krzmbrzl
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-12T09:05:21Z
user: login: Krzmbrzl
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-12T09:18:55Z

Review comments:
diff_hunk: @@ -1,7 +1,9 @@
 include(soci_define_backend_target)
 
 if (SOCI_SQLITE3_AUTO)
path: src/backends/sqlite3/CMakeLists.txt
user: login: vadz
body: @Krzmbrzl Somehow this test doesn't seem to work when rerunning CMake. I.e. it works fine when running it the first time or with `--fresh` later but not when rerunning it without that option, including when `cmake --build` reruns it automatically. I've tried to understand what's going on here, but the mechanics of defining `${BACKEND}_AUTO` are too confusing, so I'd be grateful if you could please have a look at this. TIA!
created_at: 2025-04-10T19:58:44Z
author_association: MEMBER
diff_hunk: @@ -1,7 +1,9 @@
 include(soci_define_backend_target)
 
 if (SOCI_SQLITE3_AUTO)
path: src/backends/sqlite3/CMakeLists.txt
user: login: vadz
body: OK, so this seems to happen because we intentionally override the cached variable with `ON`. I don't think we need to do this, so I'll create a separate PR to stop doing it.
created_at: 2025-04-11T00:06:27Z
author_association: MEMBER
diff_hunk: @@ -31,19 +31,27 @@
 namespace sqlite_api
 {
 
-#if SQLITE_VERSION_NUMBER < 3003010
-// The sqlite3_destructor_type typedef introduced in 3.3.10
-// https://www.sqlite.org/cvstrac/tktview?tn=2191
-typedef void (*sqlite3_destructor_type)(void*);
+// Don't include sqlite3.h from outside SOCI: this header might not be
+// available when using built-in SQLite3 as we don't install it in this case.
path: include/soci/sqlite3/soci-sqlite3.h
user: login: Krzmbrzl
body: Not installing SQLite most likely means that the install tree will be broken as the installed SOCI version will check for required packages and SQLite won't be found in that case.

Do we statically link SQLite when using the builtin version? If so, the SQLite library must not be added to the public dependencies of soci_sqlite3. Plus, we must either install the header file(s) or make sure to not include any SQLite header when using the installed SOCI lib.
created_at: 2025-04-11T07:20:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -31,19 +31,27 @@
 namespace sqlite_api
 {
 
-#if SQLITE_VERSION_NUMBER < 3003010
-// The sqlite3_destructor_type typedef introduced in 3.3.10
-// https://www.sqlite.org/cvstrac/tktview?tn=2191
-typedef void (*sqlite3_destructor_type)(void*);
+// Don't include sqlite3.h from outside SOCI: this header might not be
+// available when using built-in SQLite3 as we don't install it in this case.
path: include/soci/sqlite3/soci-sqlite3.h
user: login: vadz
body: I don't see why installing SOCI should provide SQLite, this has never been in the contract (just as installing it doesn't provide PostgreSQL or Oracle or whatever). And I think it would be wrong to install a header of a different library as part of SOCI installation.

`soci_sqlite3_builtin` is a static library and is a `PRIVATE` dependency of `soci_sqlite3`, so I think this is already fine.
created_at: 2025-04-11T13:41:19Z
author_association: MEMBER
diff_hunk: @@ -31,19 +31,27 @@
 namespace sqlite_api
 {
 
-#if SQLITE_VERSION_NUMBER < 3003010
-// The sqlite3_destructor_type typedef introduced in 3.3.10
-// https://www.sqlite.org/cvstrac/tktview?tn=2191
-typedef void (*sqlite3_destructor_type)(void*);
+// Don't include sqlite3.h from outside SOCI: this header might not be
+// available when using built-in SQLite3 as we don't install it in this case.
path: include/soci/sqlite3/soci-sqlite3.h
user: login: Krzmbrzl
body: > I don't see why installing SOCI should provide SQLite

Well, that's exactly what it is doing now, isn't it?

> soci_sqlite3_builtin is a static library and is a PRIVATE dependency of soci_sqlite3, so I think this is already fine.

No, the CMake system is set up to populate dependencies (the "public deps" ) to the config file. The way you have set up SQLite, a dependency is still created and therefore when using SOCI's SQLite backend from the install tree will almost certainly look for the SQLite library. The way I understood things, this is no longer intended, is it? :thinking: 
created_at: 2025-04-11T16:21:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -31,19 +31,27 @@
 namespace sqlite_api
 {
 
-#if SQLITE_VERSION_NUMBER < 3003010
-// The sqlite3_destructor_type typedef introduced in 3.3.10
-// https://www.sqlite.org/cvstrac/tktview?tn=2191
-typedef void (*sqlite3_destructor_type)(void*);
+// Don't include sqlite3.h from outside SOCI: this header might not be
+// available when using built-in SQLite3 as we don't install it in this case.
path: include/soci/sqlite3/soci-sqlite3.h
user: login: vadz
body: > > I don't see why installing SOCI should provide SQLite
> 
> Well, that's exactly what it is doing now, isn't it?

No, of course not. Why do you think it does? It uses the existing system SQLite, it never installs it.

> > soci_sqlite3_builtin is a static library and is a PRIVATE dependency of soci_sqlite3, so I think this is already fine.
> 
> No, the CMake system is set up to populate dependencies (the "public deps" ) to the config file. The way you have set up SQLite, a dependency is still created and therefore when using SOCI's SQLite backend from the install tree will almost certainly look for the SQLite library. The way I understood things, this is no longer intended, is it? 🤔

Would it populate it even if the dependency is not found? It shouldn't...

But I have no idea how to test this, and our CI jobs don't test it either, AFAICS, do they?
created_at: 2025-04-11T16:39:56Z
author_association: MEMBER
diff_hunk: @@ -31,19 +31,27 @@
 namespace sqlite_api
 {
 
-#if SQLITE_VERSION_NUMBER < 3003010
-// The sqlite3_destructor_type typedef introduced in 3.3.10
-// https://www.sqlite.org/cvstrac/tktview?tn=2191
-typedef void (*sqlite3_destructor_type)(void*);
+// Don't include sqlite3.h from outside SOCI: this header might not be
+// available when using built-in SQLite3 as we don't install it in this case.
path: include/soci/sqlite3/soci-sqlite3.h
user: login: Krzmbrzl
body: > No, of course not. Why do you think it does? It uses the existing system SQLite, it never installs it.

Doesn't this PR add SQLite directly into SOCI itself rather than requiring it to be installed on the target system? It not being installed, I see. But I don't see how providing a built-in SQLite is not the same as SOCI providing SQLite (in cases in which it doesn't exist on the system). :eyes: 

> Would it populate it even if the dependency is not found? It shouldn't...

If the dependency is not found the system disables the entire backend, meaning it won't get installed anyway. But this semantic is now changed with the introduction of builtin support for SQLite, which can make the backend enablement succeed even if the dependency was not found.

> But I have no idea how to test this, and our CI jobs don't test it either, AFAICS, do they?

It can be tested by installing SOCI via `make install` into a given directory and then consuming it via CMake from that directory (explicitly requesting the `SQLite3` component) all the while ensuring that we're using the new builtin SQLite option. I'll do that locally and report back what I find.

We do this on CI, but only with the `Empty` component
https://github.com/SOCI/soci/blob/e337a092d49641f2ba29ccaa7eb79023d0819641/scripts/ci/build_empty.sh#L32-L33

created_at: 2025-04-12T08:51:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -32,3 +34,31 @@ soci_define_backend_target(
 if (NOT SOCI_SQLITE3)
   return()
 endif()
+
+# If SQLite3 was found, this target must have been created.
+if (NOT TARGET SQLite::SQLite3)
+  # But if it wasn't, use our built-in version, after checking that it is
+  # available.
+  if (NOT EXISTS "${PROJECT_SOURCE_DIR}/3rdparty/sqlite3/sqlite3.c")
+    message(WARNING "SQLite3 not found and built-in version not available, have you cloned SOCI repository with --recurse-submodules?")
path: src/backends/sqlite3/CMakeLists.txt
user: login: Krzmbrzl
body: Shouldn't this be a error rather than a warning? After all, if we reach this point we have enabled the backend which so far does not have the required link libraries configured. And if we don't have the submodule, this will guarantees compile and/or link errors, doesn't it?
created_at: 2025-04-12T09:05:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -31,19 +31,27 @@
 namespace sqlite_api
 {
 
-#if SQLITE_VERSION_NUMBER < 3003010
-// The sqlite3_destructor_type typedef introduced in 3.3.10
-// https://www.sqlite.org/cvstrac/tktview?tn=2191
-typedef void (*sqlite3_destructor_type)(void*);
+// Don't include sqlite3.h from outside SOCI: this header might not be
+// available when using built-in SQLite3 as we don't install it in this case.
path: include/soci/sqlite3/soci-sqlite3.h
user: login: Krzmbrzl
body: Okay turns out that by using the builtin SQLite option, we never call `soci_public_dependency` which in turn means that the CMake part dealing with auto-generating the SOCI config file never gets to know about this component. Hence, even though we install `soci_sqlite` it won't be discoverable by downstream users via the config file. In other words: the current system builds on the assumption that all installed components (except `Core`) have at least one public dependency.

While this incidentally solves the issue I was pointing out, I think we should be more intentional about it being installed or not. Installing the library but not have it usable from the install tree doesn't seem like a desirable situation. If we don't want to install SQLite along with SOCI (which seems reasonable), we should not set the SQLite backend up for installation when using the builtin SQLite library (and add a CMake warning that will tell the user that this is what's going to happen).

----

I have found another issue with the builtin version:  if we build SOCI as a static library, we can't make SQLite a private dependency as static libraries won't get linked into other static libraries. This means that `libsoci_sqlite3.a` will claim to have no dependencies for downstream users but will lead to linker errors as
```
$ nm --undefined-only libsoci_sqlite3.a
...
session.cpp.o:
...
                 U sqlite3_busy_timeout
                 U sqlite3_close
                 U sqlite3_errmsg
                 U sqlite3_exec
                 U sqlite3_free
                 U sqlite3_libversion
                 U sqlite3_libversion_number
                 U sqlite3_open_v2
...
```
that is, we still need to link to the SQLite library when linking to `libsoci_sqlite3.a`.
created_at: 2025-04-12T09:18:55Z
author_association: CONTRIBUTOR

PR comments:
user: login: vadz
created_at: 2025-04-12T14:39:28Z
body: There is indeed a problem when using static SOCI libraries.

The simplest way to solve all this would be to avoid having a separate SQLite library and just compile SQLite as part of SOCI SQLite backend itself, and this is what I had initially tried to do. Unfortunately this resulted in compilation problems due to `soci_compiler_interface` being used by the backend: the options there are not suitable for compiling C SQLite code. If we could restrict all these options (or even the use of `soci_compiler_interface` entirely) to C++ code only, this would solve all problems — can we do this somehow?
author_association: MEMBER
user: login: Krzmbrzl
created_at: 2025-04-22T13:52:17Z
body: > can we do this somehow?

you can keep the scope of all definitions local to any given target. So as long as we retain different targets for C++ and C code, this should be achievable, yes.
author_association: CONTRIBUTOR
user: login: vadz
created_at: 2025-04-22T14:50:15Z
body: Any hints about how can I make `target_compile_options(soci_compiler_interface)` apply only to C++ code? TIA!
author_association: MEMBER
user: login: Krzmbrzl
created_at: 2025-04-22T17:42:52Z
body: I think it would only work by only linking C++ targets to `soci_compiler_interface`. I don't think there are language-specific compile options for a single target.

But recently I have come to the believe that it might in fact be better to have a function that sets all desired compiler flags as private flags for a given target. This also avoids issues with unwantingly propagating options to downstream targets. It does have the disadvantage of requiring to sue said function on all internal targets individually as we specifically can't make use of CMake's transient dependency feature.
author_association: CONTRIBUTOR
user: login: vadz
created_at: 2025-04-22T17:55:16Z
body: But the problem is that I want to have C and C++ files inside the same target. I couldn't figure out a way to do it and still use `soci_compiler_interface` as it results in compiling C code with C++ warning options.

I don't know if it's better to use an interface library or a function for the options but even if switched to the latter, we'd still have no way to set the options for only the C++ files, would we?
author_association: MEMBER
user: login: vadz
created_at: 2025-04-22T17:57:54Z
body: Hmm, wait, maybe I can just use `set_source_files_properties()` to _replace_ the C++ flags with C ones for this file? Let me try it...
author_association: MEMBER
user: login: Krzmbrzl
created_at: 2025-04-22T18:25:52Z
body: I would have recommended building SQLite as a separate target, build the SOCI lib as a seaparate target and then link both of them together into a single target.

Using per source file properties would be an alternative, I guess. 
author_association: CONTRIBUTOR
user: login: vadz
created_at: 2025-04-24T23:19:31Z
body: For the record, my actual attempt to fix it is in #1243.
author_association: MEMBER