Add lakehouse options for create table statements

.github/config/extensions/iceberg.cmake

@@ -8,7 +8,7 @@ if (NOT MINGW AND NOT ${WASM_ENABLED})
     duckdb_extension_load(iceberg
 #            ${LOAD_ICEBERG_TESTS} TODO: re-enable once autoloading test is fixed
             GIT_URL https://github.com/duckdb/duckdb-iceberg
-            GIT_TAG 30a2c66f1040ffd36e117ace6e22acbdb402e79c
+            GIT_TAG b3e54a56436d6385c6820ec6843d3e927431f565
             APPLY_PATCHES
             )
 endif()

.github/patches/extensions/iceberg/file.patch

@@ -10,79 +10,4 @@ index f7b3fa02..6bedb2f6 100644
 +IcebergLogType::IcebergLogType() : LogType(NAME, LEVEL) {
  }
 
- } // namespace duckdb
-diff --git a/src/include/iceberg_logging.hpp b/src/include/iceberg_logging.hpp
-index 56532732..94ae600f 100644
---- a/src/include/iceberg_logging.hpp
-+++ b/src/include/iceberg_logging.hpp
-@@ -13,10 +13,6 @@ struct IcebergLogType : public LogType {
- 	//! Construct the log type
- 	IcebergLogType();
-
--	static LogicalType GetLogType() {
--		return LogicalType::VARCHAR;
--	}
--
- 	template <typename... ARGS>
- 	static string ConstructLogMessage(const string &str, ARGS... params) {
- 		return StringUtil::Format(str, params...);
-diff --git a/src/include/storage/iceberg_delete.hpp b/src/include/storage/iceberg_delete.hpp
-index 43eeab28..e69c5d19 100644
---- a/src/include/storage/iceberg_delete.hpp
-+++ b/src/include/storage/iceberg_delete.hpp
-@@ -76,7 +76,7 @@ public:
-
- public:
- 	// // Source interface
--	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
-+	SourceResultType GetDataInternal(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
-
- 	bool IsSource() const override {
- 		return true;
-diff --git a/src/include/storage/iceberg_insert.hpp b/src/include/storage/iceberg_insert.hpp
-index e2e1d1a2..1c98862c 100644
---- a/src/include/storage/iceberg_insert.hpp
-+++ b/src/include/storage/iceberg_insert.hpp
-@@ -51,7 +51,7 @@ public:
-
- public:
- 	// // Source interface
--	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
-+	SourceResultType GetDataInternal(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
-
- 	bool IsSource() const override {
- 		return true;
-diff --git a/src/storage/iceberg_delete.cpp b/src/storage/iceberg_delete.cpp
-index 17952f82..14f759eb 100644
---- a/src/storage/iceberg_delete.cpp
-+++ b/src/storage/iceberg_delete.cpp
-@@ -268,7 +268,7 @@ SinkFinalizeType IcebergDelete::Finalize(Pipeline &pipeline, Event &event, Clien
- //===--------------------------------------------------------------------===//
- // GetData
- //===--------------------------------------------------------------------===//
--SourceResultType IcebergDelete::GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const {
-+SourceResultType IcebergDelete::GetDataInternal(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const {
- 	auto &global_state = sink_state->Cast<IcebergDeleteGlobalState>();
- 	auto value = Value::BIGINT(NumericCast<int64_t>(global_state.total_deleted_count));
- 	chunk.SetCardinality(1);
-diff --git a/src/storage/iceberg_insert.cpp b/src/storage/iceberg_insert.cpp
-index a49ac26a..3f5f2f25 100644
---- a/src/storage/iceberg_insert.cpp
-+++ b/src/storage/iceberg_insert.cpp
-@@ -7,7 +7,6 @@
-
- #include "iceberg_multi_file_list.hpp"
-
--#include "duckdb/common/sort/partition_state.hpp"
- #include "duckdb/catalog/catalog_entry/copy_function_catalog_entry.hpp"
- #include "duckdb/main/client_data.hpp"
- #include "duckdb/planner/operator/logical_copy_to_file.hpp"
-@@ -251,7 +250,7 @@ SinkResultType IcebergInsert::Sink(ExecutionContext &context, DataChunk &chunk,
- //===--------------------------------------------------------------------===//
- // GetData
- //===--------------------------------------------------------------------===//
--SourceResultType IcebergInsert::GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const {
-+SourceResultType IcebergInsert::GetDataInternal(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const {
- 	auto &global_state = sink_state->Cast<IcebergInsertGlobalState>();
- 	auto value = Value::BIGINT(global_state.insert_count);
- 	chunk.SetCardinality(1);
+ } // namespace duckdb

.github/workflows/BundleStaticLibs.yml

@@ -37,6 +37,7 @@ concurrency:
 env:
   GH_TOKEN: ${{ secrets.GH_TOKEN }}
   OVERRIDE_GIT_DESCRIBE: ${{ inputs.override_git_describe }}
+  BRANCHES_TO_BE_CACHED: ${{ github.repository == 'duckdb/duckdb' && '["refs/heads/main", "refs/heads/v1.4-andium"]' || ''}}
 
 jobs:
   bundle-osx-static-libs:
@@ -74,7 +75,7 @@ jobs:
         uses: hendrikmuhs/ccache-action@main
         with:
           key: ${{ github.job }}
-          save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
+          save: ${{ env.BRANCHES_TO_BE_CACHED == '' || contains(env.BRANCHES_TO_BE_CACHED, github.ref) }}
 
       - name: Build
         shell: bash

.github/workflows/CodeQuality.yml

@@ -39,6 +39,7 @@ concurrency:
 
 env:
   GH_TOKEN: ${{ secrets.GH_TOKEN }}
+  BRANCHES_TO_BE_CACHED: ${{ github.repository == 'duckdb/duckdb' && '["refs/heads/main", "refs/heads/v1.4-andium"]' || ''}}
 
 jobs:
   check-draft:
@@ -136,7 +137,7 @@ jobs:
         uses: hendrikmuhs/ccache-action@main
         with:
           key: ${{ github.job }}
-          save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
+          save: ${{ env.BRANCHES_TO_BE_CACHED == '' || contains(env.BRANCHES_TO_BE_CACHED, github.ref) }}
 
       - name: Download clang-tidy-cache
         if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/feature' }}

.github/workflows/CrossVersion.yml

@@ -24,6 +24,7 @@ concurrency:
 
 env:
   GH_TOKEN: ${{ secrets.GH_TOKEN }}
+  BRANCHES_TO_BE_CACHED: ${{ github.repository == 'duckdb/duckdb' && '["refs/heads/main", "refs/heads/v1.4-andium"]' || ''}}
 
 jobs:
   osx-step-1:
@@ -58,7 +59,7 @@ jobs:
         uses: hendrikmuhs/ccache-action@main
         with:
           key: ${{ github.job }}
-          save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
+          save: ${{ env.BRANCHES_TO_BE_CACHED == '' || contains(env.BRANCHES_TO_BE_CACHED, github.ref) }}
 
       - name: Build
         shell: bash
@@ -116,7 +117,7 @@ jobs:
         uses: hendrikmuhs/ccache-action@main
         with:
           key: ${{ github.job }}
-          save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
+          save: ${{ env.BRANCHES_TO_BE_CACHED == '' || contains(env.BRANCHES_TO_BE_CACHED, github.ref) }}
 
       - name: Build
         shell: bash
@@ -223,7 +224,7 @@ jobs:
         uses: hendrikmuhs/ccache-action@main
         with:
           key: ${{ github.job }}
-          save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
+          save: ${{ env.BRANCHES_TO_BE_CACHED == '' || contains(env.BRANCHES_TO_BE_CACHED, github.ref) }}
 
       - name: Build
         shell: bash
@@ -292,7 +293,7 @@ jobs:
         uses: hendrikmuhs/ccache-action@main
         with:
           key: ${{ github.job }}
-          save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
+          save: ${{ env.BRANCHES_TO_BE_CACHED == '' || contains(env.BRANCHES_TO_BE_CACHED, github.ref) }}
 
       - name: Build
         shell: bash

.github/workflows/ExtendedTests.yml

@@ -20,8 +20,8 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CCACHE_SAVE: ${{ github.repository != 'duckdb/duckdb' }}
   BASE_BRANCH: ${{ github.base_ref || (endsWith(github.ref, '_feature') && 'feature' || 'main') }}
+  BRANCHES_TO_BE_CACHED: ${{ github.repository == 'duckdb/duckdb' && '["refs/heads/main", "refs/heads/v1.4-andium"]' || ''}}
 
 jobs:
   regression-lto-benchmark-runner:
@@ -51,7 +51,7 @@ jobs:
        uses: hendrikmuhs/ccache-action@main
        with:
          key: ${{ github.job }}
-         save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
+         save: ${{ env.BRANCHES_TO_BE_CACHED == '' || contains(env.BRANCHES_TO_BE_CACHED, github.ref) }}
 
      - name: Build
        shell: bash
@@ -131,7 +131,7 @@ jobs:
        uses: hendrikmuhs/ccache-action@main
        with:
          key: ${{ github.job }}
-         save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
+         save: ${{ env.BRANCHES_TO_BE_CACHED == '' || contains(env.BRANCHES_TO_BE_CACHED, github.ref) }}
 
      - name: Build
        shell: bash
@@ -211,7 +211,7 @@ jobs:
          uses: hendrikmuhs/ccache-action@main
          with:
            key: ${{ github.job }}
-           save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
+           save: ${{ env.BRANCHES_TO_BE_CACHED == '' || contains(env.BRANCHES_TO_BE_CACHED, github.ref) }}
 
        - name: Build
          shell: bash
@@ -296,7 +296,7 @@ jobs:
          uses: hendrikmuhs/ccache-action@main
          with:
            key: ${{ github.job }}
-           save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
+           save: ${{ env.BRANCHES_TO_BE_CACHED == '' || contains(env.BRANCHES_TO_BE_CACHED, github.ref) }}
 
        - name: Build
          shell: bash

.github/workflows/Extensions.yml

@@ -80,13 +80,15 @@ on:
       - '.github/patches/duckdb-wasm/**'
       - '.github/workflows/**'
       - '!.github/workflows/Extensions.yml'
+      - '!.github/workflows/_extension_distribution.yml'
 
 concurrency:
   group: extensions-${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}-${{ inputs.override_git_describe }}
   cancel-in-progress: true
 
 env:
   GH_TOKEN: ${{ secrets.GH_TOKEN }}
+  BRANCHES_TO_BE_CACHED: ${{ github.repository == 'duckdb/duckdb' && '["refs/heads/main", "refs/heads/v1.4-andium"]' || ''}}
 
 jobs:
   check-draft:
@@ -97,6 +99,14 @@ jobs:
       - name: Preliminary checks on CI
         run: echo "Event name is ${{ github.event_name }}"
 
+  vars:
+    runs-on: ubuntu-24.04
+    needs: check-draft
+    outputs:
+      BRANCHES_TO_BE_CACHED: ${{ env.BRANCHES_TO_BE_CACHED }}
+    steps:
+      - run: echo "Exposing env vars"
+
   # This first step loads the various extension configs from the ~/.github/config directory storing them to drive the build jobs
   load-extension-configs:
     name: Load Extension Configs
@@ -172,6 +182,7 @@ jobs:
     name: Main Extensions
     needs:
       - load-extension-configs
+      - vars
     uses: ./.github/workflows/_extension_distribution.yml
     with:
       artifact_prefix: main-extensions
@@ -181,13 +192,14 @@ jobs:
       override_tag: ${{ inputs.override_git_describe }}
       override_duckdb_version: ${{ inputs.git_ref }}
       skip_tests: ${{ inputs.skip_tests && true || false }}
-      save_cache: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
+      save_cache: ${{ needs.vars.outputs.BRANCHES_TO_BE_CACHED == '' || contains(needs.vars.outputs.BRANCHES_TO_BE_CACHED, github.ref) }}
 
   # Build the extensions from .github/config/rust_based_extensions.cmake
   rust-based-extensions:
     name: Rust-based Extensions
     needs:
       - load-extension-configs
+      - vars
     uses: ./.github/workflows/_extension_distribution.yml
     with:
       exclude_archs: ${{ needs.load-extension-configs.outputs.rust_based_extensions_exclude_archs }}
@@ -197,13 +209,14 @@ jobs:
       override_tag: ${{ inputs.override_git_describe }}
       override_duckdb_version: ${{ inputs.git_ref }}
       skip_tests: ${{ inputs.skip_tests && true || false }}
-      save_cache: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
+      save_cache: ${{ needs.vars.outputs.BRANCHES_TO_BE_CACHED == '' || contains(needs.vars.outputs.BRANCHES_TO_BE_CACHED, github.ref) }}
 
   # Build the extensions from .github/config/external_extensions.cmake
   external-extensions:
     name: External Extensions
     needs:
       - load-extension-configs
+      - vars
     uses: ./.github/workflows/_extension_distribution.yml
     with:
       exclude_archs: ${{ needs.load-extension-configs.outputs.external_extensions_exclude_archs }}
@@ -213,7 +226,7 @@ jobs:
       override_tag: ${{ inputs.override_git_describe }}
       override_duckdb_version: ${{ inputs.git_ref }}
       skip_tests: ${{ inputs.skip_tests && true || false }}
-      save_cache: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
+      save_cache: ${{ needs.vars.outputs.BRANCHES_TO_BE_CACHED == '' || contains(needs.vars.outputs.BRANCHES_TO_BE_CACHED, github.ref) }}
 
   # Merge all extensions into a single, versioned repository
   create-extension-repository:
@@ -322,7 +335,7 @@ jobs:
         uses: hendrikmuhs/ccache-action@main
         with:
           key: ${{ github.job }}
-          save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
+          save: ${{ env.BRANCHES_TO_BE_CACHED == '' || contains(env.BRANCHES_TO_BE_CACHED, github.ref) }}
 
       - uses: actions/download-artifact@v4
         with:
@@ -380,7 +393,7 @@ jobs:
         uses: hendrikmuhs/ccache-action@main
         with:
           key: ${{ github.job }}
-          save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
+          save: ${{ env.BRANCHES_TO_BE_CACHED == '' || contains(env.BRANCHES_TO_BE_CACHED, github.ref) }}
 
       - name: Build
         shell: bash

.github/workflows/ExtraTests.yml

@@ -8,6 +8,7 @@ concurrency:
 
 env:
   GH_TOKEN: ${{ secrets.GH_TOKEN }}
+  BRANCHES_TO_BE_CACHED: ${{ github.repository == 'duckdb/duckdb' && '["refs/heads/main", "refs/heads/v1.4-andium"]' || ''}}
 
 jobs:
  regression-test-all:
@@ -38,7 +39,7 @@ jobs:
       uses: hendrikmuhs/ccache-action@main
       with:
         key: ${{ github.job }}
-        save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
+        save: ${{ env.BRANCHES_TO_BE_CACHED == '' || contains(env.BRANCHES_TO_BE_CACHED, github.ref) }}
 
     - name: Build Last Release
       shell: bash

.github/workflows/Julia.yml

@@ -37,6 +37,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
   cancel-in-progress: true
 
+env:
+  BRANCHES_TO_BE_CACHED: ${{ github.repository == 'duckdb/duckdb' && '["refs/heads/main", "refs/heads/v1.4-andium"]' || ''}}
+
 jobs:
   check-draft:
     # We run all other jobs on PRs only if they are not draft PR
@@ -97,7 +100,7 @@ jobs:
         uses: hendrikmuhs/ccache-action@main
         with:
           key: ${{ github.job }}-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.version }}
-          save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
+          save: ${{ env.BRANCHES_TO_BE_CACHED == '' || contains(env.BRANCHES_TO_BE_CACHED, github.ref) }}
 
       - name: Build DuckDB
         shell: bash

.github/workflows/LinuxRelease.yml

@@ -52,6 +52,7 @@ concurrency:
 env:
   GH_TOKEN: ${{ secrets.GH_TOKEN }}
   OVERRIDE_GIT_DESCRIBE: ${{ inputs.override_git_describe }}
+  BRANCHES_TO_BE_CACHED: ${{ github.repository == 'duckdb/duckdb' && '["refs/heads/main", "refs/heads/v1.4-andium"]' || ''}}
 
 jobs:
  check-draft:
@@ -200,7 +201,7 @@ jobs:
       uses: hendrikmuhs/ccache-action@main
       with:
         key: ${{ github.job }}
-        save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }}
+        save: ${{ env.BRANCHES_TO_BE_CACHED == '' || contains(env.BRANCHES_TO_BE_CACHED, github.ref) }}
 
     - name: Build
       shell: bash