Optimizer remove unnecessary projections

CMakeLists.txt

@@ -193,7 +193,7 @@
 else()
   if(APPLE AND CMAKE_SYSTEM_PROCESSOR MATCHES "arm64")
     if("${CMAKE_CXX_COMPILER_VERSION}" VERSION_GREATER 14.0)
       message(
         WARNING
           "Not disabling vptr sanitizer on M1 Macbook - set DISABLE_VPTR_SANITIZER manually if you run into issues with false positives in the sanitizer"
       )
@@ -203,7 +203,7 @@
   endif()
 endif()
 
 if(${ENABLE_UBSAN})
   if(${ENABLE_THREAD_SANITIZER})
     message(
       WARNING
@@ -1065,7 +1065,7 @@
 macro(register_external_extension NAME URL COMMIT DONT_LINK DONT_BUILD LOAD_TESTS PATH INCLUDE_PATH TEST_PATH APPLY_PATCHES LINKED_LIBS SUBMODULES EXTENSION_VERSION)
   include(FetchContent)
   if (${APPLY_PATCHES})
-    set(PATCH_COMMAND python3 ${CMAKE_SOURCE_DIR}/scripts/apply_extension_patches.py ${CMAKE_SOURCE_DIR}/.github/patches/extensions/${NAME}/)
+    set(PATCH_COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/scripts/apply_extension_patches.py ${CMAKE_SOURCE_DIR}/.github/patches/extensions/${NAME}/)
   endif()
   FETCHCONTENT_DECLARE(
           ${NAME}_extension_fc
@@ -1389,7 +1389,7 @@
 
   add_custom_target(
           duckdb_merge_vcpkg_manifests ALL
-          COMMAND  python3 scripts/merge_vcpkg_deps.py ${VCPKG_PATHS} ${EXT_NAMES}
+          COMMAND  ${Python3_EXECUTABLE} scripts/merge_vcpkg_deps.py ${VCPKG_PATHS} ${EXT_NAMES}
           WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
           COMMENT Generates a shared vcpkg manifest from the individual extensions)
   string(REPLACE ";"  ", " VCPKG_NAMES_COMMAS "${VCPKG_NAMES}")
@@ -1420,6 +1420,15 @@
     set(ALL_COMPILE_FLAGS "${CMAKE_CXX_FLAGS}")
   endif()
 
+  # Check for MSVC compiler and set the correct C++ standard flag
+  if(MSVC)
+    # MSVC does not support `-std=c++11` or `-std=c++14`, use `/std:c++14`
+    set(ALL_COMPILE_FLAGS "${ALL_COMPILE_FLAGS} /std:c++14")
+  else()
+    # For non-MSVC compilers, use the `-std=c++11`
+    set(ALL_COMPILE_FLAGS "${ALL_COMPILE_FLAGS} -std=c++11")
+  endif()
+
   get_target_property(duckdb_libs duckdb LINK_LIBRARIES)
 
   set(PIP_COMMAND
@@ -1432,9 +1441,9 @@
     )
 
   if(PYTHON_EDITABLE_BUILD)
-    set(PIP_COMMAND ${PIP_COMMAND} python3 -m pip install --editable .)
+    set(PIP_COMMAND ${PIP_COMMAND} ${Python3_EXECUTABLE} -m pip install --editable .)
   else()
-    set(PIP_COMMAND ${PIP_COMMAND} python3 -m pip install .)
+    set(PIP_COMMAND ${PIP_COMMAND} ${Python3_EXECUTABLE} -m pip install .)
   endif()
 
   if(USER_SPACE)

benchmark/parquet/clickbench_write.benchmark

@@ -0,0 +1,23 @@
+# name: benchmark/parquet/clickbench_write.benchmark
+# description: Write ClickBench data to Parquet
+# group: [parquet]
+
+require httpfs
+
+require parquet
+
+name ClickBench Write Parquet
+group Clickbench
+
+cache clickbench.duckdb
+
+load benchmark/clickbench/queries/load.sql
+
+init
+set preserve_insertion_order=false;
+
+run
+COPY hits TO '${BENCHMARK_DIR}/hits.parquet';
+
+result I
+10000000

data/csv/flights.csv

@@ -0,0 +1,4 @@
+FlightDate|UniqueCarrier|OriginCityName|DestCityName
+1988-01-01|AA|New York, NY|Los Angeles, CA
+1988-01-02|AA|New York, NY|Los Angeles, CA
+1988-01-03|AA|New York, NY|Los Angeles, CA

extension/core_functions/aggregate/distributive/arg_min_max.cpp

@@ -545,8 +545,8 @@ class ArgMinMaxNState {
 	BinaryAggregateHeap<K, V, COMPARATOR> heap;
 
 	bool is_initialized = false;
-	void Initialize(idx_t nval) {
-		heap.Initialize(nval);
+	void Initialize(ArenaAllocator &allocator, idx_t nval) {
+		heap.Initialize(allocator, nval);
 		is_initialized = true;
 	}
 };
@@ -601,7 +601,7 @@ static void ArgMinMaxNUpdate(Vector inputs[], AggregateInputData &aggr_input, id
 			if (nval >= MAX_N) {
 				throw InvalidInputException("Invalid input for arg_min/arg_max: n value must be < %d", MAX_N);
 			}
-			state.Initialize(UnsafeNumericCast<idx_t>(nval));
+			state.Initialize(aggr_input.allocator, UnsafeNumericCast<idx_t>(nval));
 		}
 
 		// Now add the input to the heap

extension/core_functions/scalar/generic/least.cpp

@@ -2,6 +2,7 @@
 #include "core_functions/scalar/generic_functions.hpp"
 #include "duckdb/function/create_sort_key.hpp"
 #include "duckdb/planner/expression/bound_function_expression.hpp"
+#include "duckdb/planner/expression_binder.hpp"
 
 namespace duckdb {
 

extension/json/include/json_scan.hpp

@@ -360,6 +360,8 @@ struct JSONScan {
 	                      const TableFunction &function);
 	static unique_ptr<FunctionData> Deserialize(Deserializer &deserializer, TableFunction &function);
 
+	static virtual_column_map_t GetVirtualColumns(ClientContext &context, optional_ptr<FunctionData> bind_data);
+
 	static void TableFunctionDefaults(TableFunction &table_function);
 };
 

extension/json/json_scan.cpp

@@ -171,7 +171,7 @@ unique_ptr<GlobalTableFunctionState> JSONGlobalTableFunctionState::Init(ClientCo
 		const auto &col_id = input.column_ids[col_idx];
 
 		// Skip any multi-file reader / row id stuff
-		if (col_id == bind_data.reader_bind.filename_idx || IsRowIdColumnId(col_id)) {
+		if (col_id == bind_data.reader_bind.filename_idx || IsVirtualColumn(col_id)) {
 			continue;
 		}
 		bool skip = false;
@@ -1025,6 +1025,14 @@ unique_ptr<FunctionData> JSONScan::Deserialize(Deserializer &deserializer, Table
 	return std::move(result);
 }
 
+virtual_column_map_t JSONScan::GetVirtualColumns(ClientContext &context, optional_ptr<FunctionData> bind_data) {
+	auto &csv_bind = bind_data->Cast<JSONScanData>();
+	virtual_column_map_t result;
+	MultiFileReader::GetVirtualColumns(context, csv_bind.reader_bind, result);
+	result.insert(make_pair(COLUMN_IDENTIFIER_EMPTY, TableColumn("", LogicalType::BOOLEAN)));
+	return result;
+}
+
 void JSONScan::TableFunctionDefaults(TableFunction &table_function) {
 	MultiFileReader().AddParameters(table_function);
 
@@ -1039,6 +1047,7 @@ void JSONScan::TableFunctionDefaults(TableFunction &table_function) {
 
 	table_function.serialize = Serialize;
 	table_function.deserialize = Deserialize;
+	table_function.get_virtual_columns = GetVirtualColumns;
 
 	table_function.projection_pushdown = true;
 	table_function.filter_pushdown = false;

extension/parquet/column_reader.cpp

@@ -304,7 +304,8 @@ void ColumnReader::PreparePageV2(PageHeader &page_hdr) {
 
 	auto compressed_bytes = page_hdr.compressed_page_size - uncompressed_bytes;
 
-	AllocateCompressed(compressed_bytes);
+	ResizeableBuffer compressed_buffer;
+	compressed_buffer.resize(GetAllocator(), compressed_bytes);
 	reader.ReadData(*protocol, compressed_buffer.ptr, compressed_bytes);
 
 	DecompressInternal(chunk->meta_data.codec, compressed_buffer.ptr, compressed_bytes, block->ptr + uncompressed_bytes,
@@ -319,10 +320,6 @@ void ColumnReader::AllocateBlock(idx_t size) {
 	}
 }
 
-void ColumnReader::AllocateCompressed(idx_t size) {
-	compressed_buffer.resize(GetAllocator(), size);
-}
-
 void ColumnReader::PreparePage(PageHeader &page_hdr) {
 	AllocateBlock(page_hdr.uncompressed_page_size + 1);
 	if (chunk->meta_data.codec == CompressionCodec::UNCOMPRESSED) {
@@ -333,7 +330,8 @@ void ColumnReader::PreparePage(PageHeader &page_hdr) {
 		return;
 	}
 
-	AllocateCompressed(page_hdr.compressed_page_size + 1);
+	ResizeableBuffer compressed_buffer;
+	compressed_buffer.resize(GetAllocator(), page_hdr.compressed_page_size + 1);
 	reader.ReadData(*protocol, compressed_buffer.ptr, page_hdr.compressed_page_size);
 
 	DecompressInternal(chunk->meta_data.codec, compressed_buffer.ptr, page_hdr.compressed_page_size, block->ptr,