namespace arrow {  struct Compression {   /// \brief Compression algorithm-  enum type { UNCOMPRESSED, SNAPPY, GZIP, BROTLI, ZSTD, LZ4, LZ4_FRAME, LZO, BZ2 };+  enum type {

 class ArrowColumnWriterV2 {               std::shared_ptr<Array> values_array =                   result.leaf_array->Slice(range.start, range.Size()); -              return column_writer->WriteArrow(result.def_levels, result.rep_levels,-                                               result.def_rep_level_count, *values_array,-                                               ctx);+              PARQUET_CATCH_AND_RETURN(column_writer->WriteArrow(+                  result.def_levels, result.rep_levels, result.def_rep_level_count,+                  *values_array, ctx, level_builder->Nested(), result.leaf_is_nullable));

 class TypedColumnWriterImpl : public ColumnWriterImpl, public TypedColumnWriter<     return values_to_write;   } +  void MaybeCalculateValidityBits(const int16_t* def_levels, int64_t batch_size,+                                  int64_t* out_values_to_write,+                                  int64_t* out_spaced_values_to_write,+                                  int64_t* null_count) {+    if (bits_buffer_ == nullptr) {+      if (!level_info_.HasNullableValues()) {+        *out_values_to_write = batch_size;+        *out_spaced_values_to_write = batch_size;+        *null_count = 0;+      } else {+        for (int x = 0; x < batch_size; x++) {+          *out_values_to_write += def_levels[x] == level_info_.def_level ? 1 : 0;+          *out_spaced_values_to_write +=+              def_levels[x] >= level_info_.repeated_ancestor_def_level ? 1 : 0;+        }+        *null_count = *out_values_to_write - *out_spaced_values_to_write;+      }+      return;+    }+    // Shrink to fit possible causes another allocation, and would only be necessary+    // on the last batch.+    int64_t new_bitmap_size = BitUtil::BytesForBits(batch_size);+    if (new_bitmap_size != bits_buffer_->size()) {+      PARQUET_THROW_NOT_OK(+          bits_buffer_->Resize(new_bitmap_size, /*shrink_to_fit=*/false));+      bits_buffer_->ZeroPadding();+    }+    internal::ValidityBitmapInputOutput io;+    io.valid_bits = bits_buffer_->mutable_data();+    io.values_read_upper_bound = batch_size;+    internal::DefLevelsToBitmap(def_levels, batch_size, level_info_, &io);+    *out_values_to_write = io.values_read - io.null_count;+    *out_spaced_values_to_write = io.values_read;+    *null_count = io.null_count;+  }++  std::shared_ptr<Array> MaybeUpdateArray(std::shared_ptr<Array> array,+                                          int64_t new_null_count) {+    if (bits_buffer_ == nullptr) {+      return array;+    }+    std::vector<std::shared_ptr<Buffer>> buffers = array->data()->buffers;+    buffers[0] = bits_buffer_;+    DCHECK(array->num_fields() == 0);+    return arrow::MakeArray(std::make_shared<ArrayData>(+        array->type(), array->length(), std::move(buffers), new_null_count));

 class TypedColumnWriterImpl : public ColumnWriterImpl, public TypedColumnWriter<     return values_to_write;   } +  void MaybeCalculateValidityBits(const int16_t* def_levels, int64_t batch_size,+                                  int64_t* out_values_to_write,+                                  int64_t* out_spaced_values_to_write,+                                  int64_t* null_count) {+    if (bits_buffer_ == nullptr) {+      if (!level_info_.HasNullableValues()) {+        *out_values_to_write = batch_size;+        *out_spaced_values_to_write = batch_size;+        *null_count = 0;+      } else {+        for (int x = 0; x < batch_size; x++) {+          *out_values_to_write += def_levels[x] == level_info_.def_level ? 1 : 0;+          *out_spaced_values_to_write +=+              def_levels[x] >= level_info_.repeated_ancestor_def_level ? 1 : 0;+        }+        *null_count = *out_values_to_write - *out_spaced_values_to_write;+      }+      return;+    }+    // Shrink to fit possible causes another allocation, and would only be necessary+    // on the last batch.+    int64_t new_bitmap_size = BitUtil::BytesForBits(batch_size);+    if (new_bitmap_size != bits_buffer_->size()) {+      PARQUET_THROW_NOT_OK(+          bits_buffer_->Resize(new_bitmap_size, /*shrink_to_fit=*/false));+      bits_buffer_->ZeroPadding();+    }+    internal::ValidityBitmapInputOutput io;+    io.valid_bits = bits_buffer_->mutable_data();+    io.values_read_upper_bound = batch_size;+    internal::DefLevelsToBitmap(def_levels, batch_size, level_info_, &io);+    *out_values_to_write = io.values_read - io.null_count;+    *out_spaced_values_to_write = io.values_read;+    *null_count = io.null_count;+  }++  std::shared_ptr<Array> MaybeUpdateArray(std::shared_ptr<Array> array,

 class MultipathLevelBuilderImpl : public MultipathLevelBuilder {                      std::move(write_leaf_callback));   } +  bool Nested() const override { return !data_->child_data.empty(); }

 TEST(ArrowReadWrite, SingleColumnNullableStruct) {       3); } +TEST(ArrowReadWrite, DisagreeingValidityBitmap) {}++TEST(ArrowReadWrite, NestedRequiredField) {+  auto int_field = ::arrow::field("int_array", ::arrow::int32(), /*nullable=*/false);+  auto int_array = ::arrow::ArrayFromJSON(int_field->type(), "[0, 1, 2, 3, 4, 5, 7, 8]");+  auto struct_field =+      ::arrow::field("root", ::arrow::struct_({int_field}), /*nullable=*/true);+  std::shared_ptr<Buffer> validity_bitmap;+  ASSERT_OK_AND_ASSIGN(validity_bitmap, ::arrow::AllocateBitmap(8));+  validity_bitmap->mutable_data()[0] = 0xCC;++  auto struct_data = std::make_shared<ArrayData>(+      struct_field->type(), /*length=*/8,+      std::vector<std::shared_ptr<Buffer>>{validity_bitmap},+      std::vector<std::shared_ptr<ArrayData>>{int_array->data()});+  CheckSimpleRoundtrip(+      ::arrow::Table::Make(+          ::arrow::schema({struct_field}),+          {std::make_shared<::arrow::ChunkedArray>(::arrow::MakeArray(struct_data))}),+      /*row_group_size=*/8);+}++TEST(ArrowReadWrite, NestedNullableField) {+  auto int_field = ::arrow::field("int_array", ::arrow::int32());+  auto int_array = ::arrow::ArrayFromJSON(int_field->type(), "[0, null, 2, null, 4, 5, null, 8]");+  auto struct_field =+      ::arrow::field("root", ::arrow::struct_({int_field}), /*nullable=*/true);+  std::shared_ptr<Buffer> validity_bitmap;+  ASSERT_OK_AND_ASSIGN(validity_bitmap, ::arrow::AllocateBitmap(8));+  validity_bitmap->mutable_data()[0] = 0xCC;++  auto struct_data = std::make_shared<ArrayData>(+      struct_field->type(), /*length=*/8,+      std::vector<std::shared_ptr<Buffer>>{validity_bitmap},+      std::vector<std::shared_ptr<ArrayData>>{int_array->data()});

 class TypedColumnWriterImpl : public ColumnWriterImpl, public TypedColumnWriter<    Status WriteArrow(const int16_t* def_levels, const int16_t* rep_levels,                     int64_t num_levels, const ::arrow::Array& array,-                    ArrowWriteContext* ctx) override {+                    ArrowWriteContext* ctx, bool nested, bool array_nullable) override {+    bool leaf_is_not_nullable = !level_info_.HasNullableValues();+    // Leaf nulls are canonical when there is only a single null element and it is at the+    // leaf.+    bool leaf_nulls_are_canonical =+        (level_info_.def_level == level_info_.repeated_ancestor_def_level + 1) &&+        array_nullable;+    bool maybe_has_nulls = nested && !(leaf_is_not_nullable || leaf_nulls_are_canonical);

 TEST(ArrowReadWrite, SingleColumnNullableStruct) {       3); } +TEST(ArrowReadWrite, DisagreeingValidityBitmap) {}++TEST(ArrowReadWrite, NestedRequiredField) {+  auto int_field = ::arrow::field("int_array", ::arrow::int32(), /*nullable=*/false);+  auto int_array = ::arrow::ArrayFromJSON(int_field->type(), "[0, 1, 2, 3, 4, 5, 7, 8]");+  auto struct_field =+      ::arrow::field("root", ::arrow::struct_({int_field}), /*nullable=*/true);+  std::shared_ptr<Buffer> validity_bitmap;+  ASSERT_OK_AND_ASSIGN(validity_bitmap, ::arrow::AllocateBitmap(8));+  validity_bitmap->mutable_data()[0] = 0xCC;++  auto struct_data = std::make_shared<ArrayData>(+      struct_field->type(), /*length=*/8,+      std::vector<std::shared_ptr<Buffer>>{validity_bitmap},+      std::vector<std::shared_ptr<ArrayData>>{int_array->data()});+  CheckSimpleRoundtrip(+      ::arrow::Table::Make(+          ::arrow::schema({struct_field}),+          {std::make_shared<::arrow::ChunkedArray>(::arrow::MakeArray(struct_data))}),+      /*row_group_size=*/8);

 cdef class Scanner(_Weakrefable):     def from_fragment(Fragment fragment not None, Schema schema=None,                       bint use_threads=True, MemoryPool memory_pool=None,                       list columns=None, Expression filter=None,-                      int batch_size=32*2**10):+                      int batch_size=2**20):

 class ARROW_DS_EXPORT ParquetFileFormat : public FileFormat {     /// @{     std::unordered_set<std::string> dict_columns;     /// @}++    /// Parallelize conversion across columns. This option is ignored if a scan is already+    /// parallelized across input files.

 cdef class ParquetReadOptions(_Weakrefable):     dictionary_columns : list of string, default None         Names of columns which should be dictionary encoded as         they are read.+    enable_parallel_column_conversion : bool, default False+        Whether single files may be read in parallel (ignored when reading+        multiple files to avoid thread contention).

 cdef shared_ptr[CScanContext] _build_scan_context(bint use_threads=True,  cdef void _populate_builder(const shared_ptr[CScannerBuilder]& ptr,                             list columns=None, Expression filter=None,-                            int batch_size=32*2**10) except *:+                            int batch_size=2**20) except *:

 class ARROW_EXPORT ExtensionType : public DataType {   /// \return the serialized representation   virtual std::string Serialize() const = 0; +  static std::shared_ptr<Array> WrapArray(const std::shared_ptr<DataType>& ext_type,+                                          const std::shared_ptr<Array>& storage);+  static std::shared_ptr<ChunkedArray> WrapArray(+      const std::shared_ptr<DataType>& ext_type,+      const std::shared_ptr<ChunkedArray>& storage);

  #include <cstdint> #include <memory>-#include <unordered_map> #include <utility> #include <vector> -#include "arrow/memory_pool.h"+#include "arrow/result.h" #include "arrow/status.h"+#include "arrow/type_fwd.h" #include "arrow/util/macros.h" #include "arrow/util/visibility.h"  namespace arrow {+namespace ipc { -class Array;-class DataType;-class Field;-class RecordBatch;+class FieldPosition {+ public:+  FieldPosition() : parent_(NULLPTR), index_(-1), depth_(0) {}

  #include <cstdint> #include <memory>-#include <unordered_map> #include <utility> #include <vector> -#include "arrow/memory_pool.h"+#include "arrow/result.h" #include "arrow/status.h"+#include "arrow/type_fwd.h" #include "arrow/util/macros.h" #include "arrow/util/visibility.h"  namespace arrow {+namespace ipc {++class FieldPosition {+ public:+  FieldPosition() : parent_(NULLPTR), index_(-1), depth_(0) {}++  FieldPosition child(int index) const { return {this, index}; }++  std::vector<int> path() const {+    std::vector<int> path(depth_);+    const FieldPosition* cur = this;+    for (int i = depth_ - 1; i >= 0; --i) {+      path[i] = cur->index_;+      cur = cur->parent_;+    }+    return path;+  }++ protected:+  FieldPosition(const FieldPosition* parent, int index)+      : parent_(parent), index_(index), depth_(parent->depth_ + 1) {}++  const FieldPosition* parent_;+  int index_;+  int depth_;+}; -class Array;-class DataType;-class Field;-class RecordBatch;+/// \brief Map fields in a schema to dictionary ids+///+/// The mapping is structural, i.e. the field path (as a vector of indices)+/// is associated to the dictionary id.+class ARROW_EXPORT DictionaryFieldMapper {+ public:+  DictionaryFieldMapper();+  explicit DictionaryFieldMapper(const Schema& schema);+  ~DictionaryFieldMapper(); -namespace ipc {+  Status AddSchemaFields(const Schema& schema);+  Status AddField(int64_t id, std::vector<int> field_path); -/// \brief Memoization data structure for assigning id numbers to-/// dictionaries and tracking their current state through possible-/// deltas in an IPC stream+  Result<int64_t> GetFieldId(std::vector<int> field_path) const;++  int num_fields() const;++ private:+  struct Impl;+  std::unique_ptr<Impl> impl_;+};++using DictionaryVector = std::vector<std::pair<int64_t, std::shared_ptr<Array>>>;++/// \brief Memoization data structure for reading dictionaries from IPC streams+///+/// This structure tracks the following associations:+/// - field position (structural) -> dictionary id+/// - dictionary id -> value type+/// - dictionary id -> dictionary (value) data+///+/// Together, they allow resolving dictionary data when reading an IPC stream,+/// using metadata recorded in the schema message and data recorded in the+/// dictionary batch messages (see ResolveDictionaries).+///+/// This structure isn't useful for writing an IPC stream, where only+/// DictionaryFieldMapper is necessary. class ARROW_EXPORT DictionaryMemo {  public:-  using DictionaryVector = std::vector<std::pair<int64_t, std::shared_ptr<Array>>>;-   DictionaryMemo();-  DictionaryMemo(DictionaryMemo&&) = default;-  DictionaryMemo& operator=(DictionaryMemo&&) = default;+  ~DictionaryMemo();++  DictionaryFieldMapper& fields();

 #include "arrow/status.h" #include "arrow/type.h" #include "arrow/util/checked_cast.h"+#include "arrow/util/logging.h"++namespace std {+template <>+struct hash<arrow::FieldPath> {+  size_t operator()(const arrow::FieldPath& path) const { return path.hash(); }+};

 namespace flatbuf = org::apache::arrow::flatbuf;  namespace ipc { +class DictionaryFieldMapper; class DictionaryMemo;

  #include <cstdint> #include <memory>-#include <unordered_map> #include <utility> #include <vector> -#include "arrow/memory_pool.h"+#include "arrow/result.h" #include "arrow/status.h"+#include "arrow/type_fwd.h" #include "arrow/util/macros.h" #include "arrow/util/visibility.h"  namespace arrow {+namespace ipc {++class FieldPosition {+ public:+  FieldPosition() : parent_(NULLPTR), index_(-1), depth_(0) {}++  FieldPosition child(int index) const { return {this, index}; }++  std::vector<int> path() const {+    std::vector<int> path(depth_);+    const FieldPosition* cur = this;+    for (int i = depth_ - 1; i >= 0; --i) {+      path[i] = cur->index_;+      cur = cur->parent_;+    }+    return path;+  }++ protected:+  FieldPosition(const FieldPosition* parent, int index)+      : parent_(parent), index_(index), depth_(parent->depth_ + 1) {}++  const FieldPosition* parent_;+  int index_;+  int depth_;+}; -class Array;-class DataType;-class Field;-class RecordBatch;+/// \brief Map fields in a schema to dictionary ids+///+/// The mapping is structural, i.e. the field path (as a vector of indices)+/// is associated to the dictionary id.

 Status FromProto(const pb::SchemaResult& pb_result, std::string* result) { }  Status SchemaToString(const Schema& schema, std::string* out) {-  // TODO(wesm): Do we care about better memory efficiency here?   ipc::DictionaryMemo unused_dict_memo;

 struct DuckDBPyResult { 		return py::module::import("pandas").attr("DataFrame").attr("from_dict")(fetchnumpy()); 	} +	py::object fetch_arrow_table() {+		if (!result) {+			throw runtime_error("result closed");+		}++		auto pyarrow_lib_module = py::module::import("pyarrow").attr("lib");++		auto batch_import_func = pyarrow_lib_module.attr("RecordBatch").attr("_import_from_c");+		auto from_batches_func = pyarrow_lib_module.attr("Table").attr("from_batches");+		auto schema_import_func = pyarrow_lib_module.attr("Schema").attr("_import_from_c");+		ArrowSchema schema;+		result->ToArrowSchema(&schema);+		auto schema_obj = schema_import_func((uint64_t)&schema);++		py::list batches;+		while (true) {+			auto data_chunk = result->Fetch();+			if (data_chunk->size() == 0) {+				break;+			}+			ArrowArray data;+			data_chunk->ToArrowArray(&data);+			ArrowSchema schema;+			result->ToArrowSchema(&schema);+			batches.append(batch_import_func((uint64_t)&data, (uint64_t)&schema));+		}+		return from_batches_func(batches, schema_obj);+	}

 string QueryResult::HeaderToString() { 	return result; } +struct DuckDBArrowSchemaHolder {+	// unused in root+	ArrowSchema schema;+	// unused in children+	unique_ptr<ArrowSchema *[]> children; // just space for the *pointers* to children, not the children themselves+};++static void release_duckdb_arrow_schema(ArrowSchema *schema) {+	if (!schema || !schema->release) {+		return;+	}+	schema->release = nullptr;+	auto holder = (DuckDBArrowSchemaHolder *)schema->private_data;+	delete holder;+}++void QueryResult::ToArrowSchema(ArrowSchema *out_schema) {+	assert(out_schema);++	auto root_holder = new DuckDBArrowSchemaHolder();++	root_holder->children = unique_ptr<ArrowSchema *[]>(new ArrowSchema *[column_count()]);+	out_schema->private_data = root_holder;+	out_schema->release = release_duckdb_arrow_schema;++	out_schema->children = root_holder->children.get();++	out_schema->format = "+s"; // struct apparently

+#include "duckdb.hpp"++#include "duckdb/common/arrow.hpp"+#include "duckdb/function/table/arrow.hpp"++#include "duckdb/parser/parsed_data/create_table_function_info.hpp"+#include "duckdb/main/client_context.hpp"+#include "duckdb/main/connection.hpp"+#include "duckdb/function/table_function.hpp"+#include "duckdb/common/types/timestamp.hpp"+#include "duckdb/common/types/date.hpp"++#include "utf8proc_wrapper.hpp"++namespace duckdb {++struct ArrowScanFunctionData : public TableFunctionData {+	ArrowArrayStream *stream;+	ArrowSchema schema_root;+	ArrowArray current_chunk_root;+	idx_t chunk_idx = 0;+	idx_t chunk_offset = 0;++	void ReleaseArray() {+		if (current_chunk_root.release) {+			for (idx_t child_idx = 0; child_idx < (idx_t)current_chunk_root.n_children; child_idx++) {+				auto &child = *current_chunk_root.children[child_idx];+				if (child.release) {+					child.release(&child);+				}+			}+			current_chunk_root.release(&current_chunk_root);+		}+	}++	void ReleaseSchema() {+		if (schema_root.release) {+			for (idx_t child_idx = 0; child_idx < (idx_t)schema_root.n_children; child_idx++) {+				auto &child = *schema_root.children[child_idx];+				if (child.release) {+					child.release(&child);+				}+			}+			schema_root.release(&schema_root);+		}+	}++	~ArrowScanFunctionData() {+		ReleaseSchema();+		ReleaseArray();+	}+};++static unique_ptr<FunctionData> arrow_scan_bind(ClientContext &context, vector<Value> &inputs,+                                                unordered_map<string, Value> &named_parameters,+                                                vector<LogicalType> &return_types, vector<string> &names) {++	auto res = make_unique<ArrowScanFunctionData>();+	auto &data = *res;++	data.stream = (ArrowArrayStream *)inputs[0].GetValue<uintptr_t>();+	if (!data.stream) {+		throw InvalidInputException("arrow_scan: NULL pointer passed");+	}++	if (data.stream->get_schema(data.stream, &data.schema_root)) {+		throw InvalidInputException("arrow_scan: get_schema failed(): %s",+		                            string(data.stream->get_last_error(data.stream)));+	}++	if (!data.schema_root.release) {+		throw InvalidInputException("arrow_scan: released schema passed");+	}++	if (data.schema_root.n_children < 1) {+		throw InvalidInputException("arrow_scan: empty schema passed");+	}++	for (idx_t col_idx = 0; col_idx < (idx_t)data.schema_root.n_children; col_idx++) {+		auto &schema = *data.schema_root.children[col_idx];+		if (!schema.release) {+			throw InvalidInputException("arrow_scan: released schema passed");+		}+		if (schema.dictionary) {+			throw NotImplementedException("arrow_scan: dictionary vectors not supported yet");+		}+		auto format = string(schema.format);+		if (format == "b") {+			return_types.push_back(LogicalType::BOOLEAN);+		} else if (format == "c") {+			return_types.push_back(LogicalType::TINYINT);+		} else if (format == "s") {+			return_types.push_back(LogicalType::SMALLINT);+		} else if (format == "i") {+			return_types.push_back(LogicalType::INTEGER);+		} else if (format == "l") {+			return_types.push_back(LogicalType::BIGINT);+		} else if (format == "f") {+			return_types.push_back(LogicalType::FLOAT);+		} else if (format == "g") {+			return_types.push_back(LogicalType::DOUBLE);+		} else if (format == "d:38,0") { // decimal128+			return_types.push_back(LogicalType::HUGEINT);+		} else if (format == "u") {+			return_types.push_back(LogicalType::VARCHAR);+		} else if (format == "tsn:") {+			return_types.push_back(LogicalType::TIMESTAMP);+		} else {+			throw NotImplementedException("Unsupported Arrow type %s", format);+		}+		auto name = string(schema.name);+		if (name.empty()) {+			name = string("v") + to_string(col_idx);+		}+		names.push_back(name);+	}+	data.ReleaseSchema();+	return move(res);+}++static void arrow_scan_function(ClientContext &context, vector<Value> &input, DataChunk &output,+                                FunctionData *dataptr) {+	auto &data = *((ArrowScanFunctionData *)dataptr);++	if (!data.stream->release) { // no more chunks+		return;+	}++	// have we run out of data on the current chunk? move to next one+	if (data.chunk_offset >= (idx_t)data.current_chunk_root.length) {+		data.chunk_offset = 0;+		data.ReleaseArray();+		if (data.stream->get_next(data.stream, &data.current_chunk_root)) {+			throw InvalidInputException("arrow_scan: get_next failed(): %s",+			                            string(data.stream->get_last_error(data.stream)));+		}+	}++	// have we run out of chunks? we done+	if (!data.current_chunk_root.release) {+		data.stream->release(data.stream);+		return;+	}++	if ((idx_t)data.current_chunk_root.n_children != output.column_count()) {+		throw InvalidInputException("arrow_scan: array column count mismatch");+	}++	output.SetCardinality(+	    std::min((int64_t)STANDARD_VECTOR_SIZE, (int64_t)(data.current_chunk_root.length - data.chunk_offset)));++	for (idx_t col_idx = 0; col_idx < output.column_count(); col_idx++) {+		auto &array = *data.current_chunk_root.children[col_idx];+		if (!array.release) {+			throw InvalidInputException("arrow_scan: released array passed");+		}+		if (array.length != data.current_chunk_root.length) {+			throw InvalidInputException("arrow_scan: array length mismatch");+		}+		if (array.dictionary) {+			throw NotImplementedException("arrow_scan: dictionary vectors not supported yet");+		}+		if (array.null_count != 0 && array.buffers[0]) {+			auto &nullmask = FlatVector::Nullmask(output.data[col_idx]);++			auto bit_offset = data.chunk_offset + array.offset;+			auto n_bitmask_bytes = (output.size() + 8 - 1) / 8;++			if (bit_offset % 8 == 0) {+				// just memcpy nullmask+				memcpy(&nullmask, (uint8_t *)array.buffers[0] + bit_offset / 8, n_bitmask_bytes);+			} else {+				// need to re-align nullmask :/+				bitset<STANDARD_VECTOR_SIZE + 8> temp_nullmask;+				memcpy(&temp_nullmask, (uint8_t *)array.buffers[0] + bit_offset / 8, n_bitmask_bytes + 1);++				temp_nullmask >>= (bit_offset % 8); // why this has to be a right shift is a mystery to me+				memcpy(&nullmask, (data_ptr_t)&temp_nullmask, n_bitmask_bytes);+			}+			nullmask.flip(); // arrow uses inverse nullmask logic+		}++		switch (output.data[col_idx].type.id()) {+		case LogicalTypeId::BOOLEAN:+		case LogicalTypeId::TINYINT:+		case LogicalTypeId::SMALLINT:+		case LogicalTypeId::INTEGER:+		case LogicalTypeId::FLOAT:+		case LogicalTypeId::DOUBLE:+		case LogicalTypeId::BIGINT:+		case LogicalTypeId::HUGEINT:+			FlatVector::SetData(output.data[col_idx],+			                    (data_ptr_t)array.buffers[1] + GetTypeIdSize(output.data[col_idx].type.InternalType()) *+			                                                       (data.chunk_offset + array.offset));+			break;++		case LogicalTypeId::VARCHAR: {+			auto offsets = (uint32_t *)array.buffers[1] + array.offset + data.chunk_offset;+			auto cdata = (char *)array.buffers[2];++			for (idx_t row_idx = 0; row_idx < output.size(); row_idx++) {+				if (FlatVector::Nullmask(output.data[col_idx])[row_idx]) {+					continue;+				}+				auto cptr = cdata + offsets[row_idx];+				auto str_len = offsets[row_idx + 1] - offsets[row_idx];++				auto utf_type = Utf8Proc::Analyze(cptr, str_len);+				switch (utf_type) {+				case UnicodeType::ASCII:+					FlatVector::GetData<string_t>(output.data[col_idx])[row_idx] =+					    StringVector::AddString(output.data[col_idx], cptr, str_len);+					break;+				case UnicodeType::UNICODE:+					// this regrettably copies to normalize+					FlatVector::GetData<string_t>(output.data[col_idx])[row_idx] =+					    StringVector::AddString(output.data[col_idx], Utf8Proc::Normalize(string(cptr, str_len)));++					break;+				case UnicodeType::INVALID:+					throw runtime_error("Invalid UTF8 string encoding");+				}+			}++			break;+		} // TODO timestamps in duckdb are subject to change+		case LogicalTypeId::TIMESTAMP: {+			auto src_ptr = (uint64_t *)array.buffers[1] + data.chunk_offset;+			auto tgt_ptr = (timestamp_t *)FlatVector::GetData(output.data[col_idx]);++			for (idx_t row = 0; row < output.size(); row++) {+				auto source_idx = data.chunk_offset + row;++				auto ms = src_ptr[source_idx] / 1000000; // nanoseconds

+#include "duckdb.hpp"++#include "duckdb/common/arrow.hpp"+#include "duckdb/function/table/arrow.hpp"++#include "duckdb/parser/parsed_data/create_table_function_info.hpp"+#include "duckdb/main/client_context.hpp"+#include "duckdb/main/connection.hpp"+#include "duckdb/function/table_function.hpp"+#include "duckdb/common/types/timestamp.hpp"+#include "duckdb/common/types/date.hpp"++#include "utf8proc_wrapper.hpp"++namespace duckdb {++struct ArrowScanFunctionData : public TableFunctionData {+	ArrowArrayStream *stream;+	ArrowSchema schema_root;+	ArrowArray current_chunk_root;+	idx_t chunk_idx = 0;+	idx_t chunk_offset = 0;++	void ReleaseArray() {+		if (current_chunk_root.release) {+			for (idx_t child_idx = 0; child_idx < (idx_t)current_chunk_root.n_children; child_idx++) {+				auto &child = *current_chunk_root.children[child_idx];+				if (child.release) {+					child.release(&child);+				}+			}+			current_chunk_root.release(&current_chunk_root);+		}+	}++	void ReleaseSchema() {+		if (schema_root.release) {+			for (idx_t child_idx = 0; child_idx < (idx_t)schema_root.n_children; child_idx++) {+				auto &child = *schema_root.children[child_idx];+				if (child.release) {+					child.release(&child);+				}+			}+			schema_root.release(&schema_root);+		}+	}++	~ArrowScanFunctionData() {+		ReleaseSchema();+		ReleaseArray();+	}+};++static unique_ptr<FunctionData> arrow_scan_bind(ClientContext &context, vector<Value> &inputs,+                                                unordered_map<string, Value> &named_parameters,+                                                vector<LogicalType> &return_types, vector<string> &names) {++	auto res = make_unique<ArrowScanFunctionData>();+	auto &data = *res;++	data.stream = (ArrowArrayStream *)inputs[0].GetValue<uintptr_t>();+	if (!data.stream) {+		throw InvalidInputException("arrow_scan: NULL pointer passed");+	}++	if (data.stream->get_schema(data.stream, &data.schema_root)) {+		throw InvalidInputException("arrow_scan: get_schema failed(): %s",+		                            string(data.stream->get_last_error(data.stream)));+	}++	if (!data.schema_root.release) {+		throw InvalidInputException("arrow_scan: released schema passed");+	}++	if (data.schema_root.n_children < 1) {+		throw InvalidInputException("arrow_scan: empty schema passed");+	}++	for (idx_t col_idx = 0; col_idx < (idx_t)data.schema_root.n_children; col_idx++) {+		auto &schema = *data.schema_root.children[col_idx];+		if (!schema.release) {+			throw InvalidInputException("arrow_scan: released schema passed");+		}+		if (schema.dictionary) {+			throw NotImplementedException("arrow_scan: dictionary vectors not supported yet");+		}+		auto format = string(schema.format);+		if (format == "b") {+			return_types.push_back(LogicalType::BOOLEAN);+		} else if (format == "c") {+			return_types.push_back(LogicalType::TINYINT);+		} else if (format == "s") {+			return_types.push_back(LogicalType::SMALLINT);+		} else if (format == "i") {+			return_types.push_back(LogicalType::INTEGER);+		} else if (format == "l") {+			return_types.push_back(LogicalType::BIGINT);+		} else if (format == "f") {+			return_types.push_back(LogicalType::FLOAT);+		} else if (format == "g") {+			return_types.push_back(LogicalType::DOUBLE);+		} else if (format == "d:38,0") { // decimal128+			return_types.push_back(LogicalType::HUGEINT);+		} else if (format == "u") {+			return_types.push_back(LogicalType::VARCHAR);+		} else if (format == "tsn:") {+			return_types.push_back(LogicalType::TIMESTAMP);+		} else {+			throw NotImplementedException("Unsupported Arrow type %s", format);+		}+		auto name = string(schema.name);+		if (name.empty()) {+			name = string("v") + to_string(col_idx);+		}+		names.push_back(name);+	}+	data.ReleaseSchema();+	return move(res);+}++static void arrow_scan_function(ClientContext &context, vector<Value> &input, DataChunk &output,+                                FunctionData *dataptr) {+	auto &data = *((ArrowScanFunctionData *)dataptr);++	if (!data.stream->release) { // no more chunks+		return;+	}++	// have we run out of data on the current chunk? move to next one+	if (data.chunk_offset >= (idx_t)data.current_chunk_root.length) {+		data.chunk_offset = 0;+		data.ReleaseArray();+		if (data.stream->get_next(data.stream, &data.current_chunk_root)) {+			throw InvalidInputException("arrow_scan: get_next failed(): %s",+			                            string(data.stream->get_last_error(data.stream)));+		}+	}++	// have we run out of chunks? we done+	if (!data.current_chunk_root.release) {+		data.stream->release(data.stream);+		return;+	}++	if ((idx_t)data.current_chunk_root.n_children != output.column_count()) {+		throw InvalidInputException("arrow_scan: array column count mismatch");+	}++	output.SetCardinality(+	    std::min((int64_t)STANDARD_VECTOR_SIZE, (int64_t)(data.current_chunk_root.length - data.chunk_offset)));++	for (idx_t col_idx = 0; col_idx < output.column_count(); col_idx++) {+		auto &array = *data.current_chunk_root.children[col_idx];+		if (!array.release) {+			throw InvalidInputException("arrow_scan: released array passed");+		}+		if (array.length != data.current_chunk_root.length) {+			throw InvalidInputException("arrow_scan: array length mismatch");+		}+		if (array.dictionary) {+			throw NotImplementedException("arrow_scan: dictionary vectors not supported yet");+		}+		if (array.null_count != 0 && array.buffers[0]) {+			auto &nullmask = FlatVector::Nullmask(output.data[col_idx]);++			auto bit_offset = data.chunk_offset + array.offset;+			auto n_bitmask_bytes = (output.size() + 8 - 1) / 8;++			if (bit_offset % 8 == 0) {+				// just memcpy nullmask+				memcpy(&nullmask, (uint8_t *)array.buffers[0] + bit_offset / 8, n_bitmask_bytes);+			} else {+				// need to re-align nullmask :/+				bitset<STANDARD_VECTOR_SIZE + 8> temp_nullmask;+				memcpy(&temp_nullmask, (uint8_t *)array.buffers[0] + bit_offset / 8, n_bitmask_bytes + 1);++				temp_nullmask >>= (bit_offset % 8); // why this has to be a right shift is a mystery to me+				memcpy(&nullmask, (data_ptr_t)&temp_nullmask, n_bitmask_bytes);+			}+			nullmask.flip(); // arrow uses inverse nullmask logic+		}++		switch (output.data[col_idx].type.id()) {+		case LogicalTypeId::BOOLEAN:+		case LogicalTypeId::TINYINT:+		case LogicalTypeId::SMALLINT:+		case LogicalTypeId::INTEGER:+		case LogicalTypeId::FLOAT:+		case LogicalTypeId::DOUBLE:+		case LogicalTypeId::BIGINT:+		case LogicalTypeId::HUGEINT:+			FlatVector::SetData(output.data[col_idx],+			                    (data_ptr_t)array.buffers[1] + GetTypeIdSize(output.data[col_idx].type.InternalType()) *+			                                                       (data.chunk_offset + array.offset));+			break;++		case LogicalTypeId::VARCHAR: {+			auto offsets = (uint32_t *)array.buffers[1] + array.offset + data.chunk_offset;+			auto cdata = (char *)array.buffers[2];++			for (idx_t row_idx = 0; row_idx < output.size(); row_idx++) {+				if (FlatVector::Nullmask(output.data[col_idx])[row_idx]) {+					continue;+				}+				auto cptr = cdata + offsets[row_idx];+				auto str_len = offsets[row_idx + 1] - offsets[row_idx];++				auto utf_type = Utf8Proc::Analyze(cptr, str_len);+				switch (utf_type) {+				case UnicodeType::ASCII:+					FlatVector::GetData<string_t>(output.data[col_idx])[row_idx] =+					    StringVector::AddString(output.data[col_idx], cptr, str_len);+					break;+				case UnicodeType::UNICODE:+					// this regrettably copies to normalize

+#include "duckdb.hpp"++#include "duckdb/common/arrow.hpp"+#include "duckdb/function/table/arrow.hpp"++#include "duckdb/parser/parsed_data/create_table_function_info.hpp"+#include "duckdb/main/client_context.hpp"+#include "duckdb/main/connection.hpp"+#include "duckdb/function/table_function.hpp"+#include "duckdb/common/types/timestamp.hpp"+#include "duckdb/common/types/date.hpp"++#include "utf8proc_wrapper.hpp"++namespace duckdb {++struct ArrowScanFunctionData : public TableFunctionData {+	ArrowArrayStream *stream;+	ArrowSchema schema_root;+	ArrowArray current_chunk_root;+	idx_t chunk_idx = 0;+	idx_t chunk_offset = 0;++	void ReleaseArray() {+		if (current_chunk_root.release) {+			for (idx_t child_idx = 0; child_idx < (idx_t)current_chunk_root.n_children; child_idx++) {+				auto &child = *current_chunk_root.children[child_idx];+				if (child.release) {+					child.release(&child);+				}+			}+			current_chunk_root.release(&current_chunk_root);+		}+	}++	void ReleaseSchema() {+		if (schema_root.release) {+			for (idx_t child_idx = 0; child_idx < (idx_t)schema_root.n_children; child_idx++) {+				auto &child = *schema_root.children[child_idx];+				if (child.release) {+					child.release(&child);+				}+			}+			schema_root.release(&schema_root);+		}+	}++	~ArrowScanFunctionData() {+		ReleaseSchema();+		ReleaseArray();+	}+};++static unique_ptr<FunctionData> arrow_scan_bind(ClientContext &context, vector<Value> &inputs,+                                                unordered_map<string, Value> &named_parameters,+                                                vector<LogicalType> &return_types, vector<string> &names) {++	auto res = make_unique<ArrowScanFunctionData>();+	auto &data = *res;++	data.stream = (ArrowArrayStream *)inputs[0].GetValue<uintptr_t>();+	if (!data.stream) {+		throw InvalidInputException("arrow_scan: NULL pointer passed");+	}++	if (data.stream->get_schema(data.stream, &data.schema_root)) {+		throw InvalidInputException("arrow_scan: get_schema failed(): %s",+		                            string(data.stream->get_last_error(data.stream)));+	}++	if (!data.schema_root.release) {+		throw InvalidInputException("arrow_scan: released schema passed");+	}++	if (data.schema_root.n_children < 1) {+		throw InvalidInputException("arrow_scan: empty schema passed");+	}++	for (idx_t col_idx = 0; col_idx < (idx_t)data.schema_root.n_children; col_idx++) {+		auto &schema = *data.schema_root.children[col_idx];+		if (!schema.release) {+			throw InvalidInputException("arrow_scan: released schema passed");+		}+		if (schema.dictionary) {+			throw NotImplementedException("arrow_scan: dictionary vectors not supported yet");+		}+		auto format = string(schema.format);+		if (format == "b") {+			return_types.push_back(LogicalType::BOOLEAN);+		} else if (format == "c") {+			return_types.push_back(LogicalType::TINYINT);+		} else if (format == "s") {+			return_types.push_back(LogicalType::SMALLINT);+		} else if (format == "i") {+			return_types.push_back(LogicalType::INTEGER);+		} else if (format == "l") {+			return_types.push_back(LogicalType::BIGINT);+		} else if (format == "f") {+			return_types.push_back(LogicalType::FLOAT);+		} else if (format == "g") {+			return_types.push_back(LogicalType::DOUBLE);+		} else if (format == "d:38,0") { // decimal128+			return_types.push_back(LogicalType::HUGEINT);+		} else if (format == "u") {+			return_types.push_back(LogicalType::VARCHAR);

+#include "duckdb.hpp"++#include "duckdb/common/arrow.hpp"+#include "duckdb/function/table/arrow.hpp"++#include "duckdb/parser/parsed_data/create_table_function_info.hpp"+#include "duckdb/main/client_context.hpp"+#include "duckdb/main/connection.hpp"+#include "duckdb/function/table_function.hpp"+#include "duckdb/common/types/timestamp.hpp"+#include "duckdb/common/types/date.hpp"++#include "utf8proc_wrapper.hpp"++namespace duckdb {++struct ArrowScanFunctionData : public TableFunctionData {+	ArrowArrayStream *stream;+	ArrowSchema schema_root;+	ArrowArray current_chunk_root;+	idx_t chunk_idx = 0;+	idx_t chunk_offset = 0;++	void ReleaseArray() {+		if (current_chunk_root.release) {+			for (idx_t child_idx = 0; child_idx < (idx_t)current_chunk_root.n_children; child_idx++) {+				auto &child = *current_chunk_root.children[child_idx];+				if (child.release) {+					child.release(&child);+				}+			}+			current_chunk_root.release(&current_chunk_root);+		}+	}++	void ReleaseSchema() {+		if (schema_root.release) {+			for (idx_t child_idx = 0; child_idx < (idx_t)schema_root.n_children; child_idx++) {+				auto &child = *schema_root.children[child_idx];+				if (child.release) {+					child.release(&child);+				}+			}+			schema_root.release(&schema_root);+		}+	}++	~ArrowScanFunctionData() {+		ReleaseSchema();+		ReleaseArray();+	}+};++static unique_ptr<FunctionData> arrow_scan_bind(ClientContext &context, vector<Value> &inputs,+                                                unordered_map<string, Value> &named_parameters,+                                                vector<LogicalType> &return_types, vector<string> &names) {++	auto res = make_unique<ArrowScanFunctionData>();+	auto &data = *res;++	data.stream = (ArrowArrayStream *)inputs[0].GetValue<uintptr_t>();+	if (!data.stream) {+		throw InvalidInputException("arrow_scan: NULL pointer passed");+	}++	if (data.stream->get_schema(data.stream, &data.schema_root)) {+		throw InvalidInputException("arrow_scan: get_schema failed(): %s",+		                            string(data.stream->get_last_error(data.stream)));+	}++	if (!data.schema_root.release) {+		throw InvalidInputException("arrow_scan: released schema passed");+	}++	if (data.schema_root.n_children < 1) {+		throw InvalidInputException("arrow_scan: empty schema passed");+	}++	for (idx_t col_idx = 0; col_idx < (idx_t)data.schema_root.n_children; col_idx++) {+		auto &schema = *data.schema_root.children[col_idx];+		if (!schema.release) {+			throw InvalidInputException("arrow_scan: released schema passed");+		}+		if (schema.dictionary) {+			throw NotImplementedException("arrow_scan: dictionary vectors not supported yet");+		}+		auto format = string(schema.format);+		if (format == "b") {+			return_types.push_back(LogicalType::BOOLEAN);+		} else if (format == "c") {+			return_types.push_back(LogicalType::TINYINT);+		} else if (format == "s") {+			return_types.push_back(LogicalType::SMALLINT);+		} else if (format == "i") {+			return_types.push_back(LogicalType::INTEGER);+		} else if (format == "l") {+			return_types.push_back(LogicalType::BIGINT);+		} else if (format == "f") {+			return_types.push_back(LogicalType::FLOAT);+		} else if (format == "g") {+			return_types.push_back(LogicalType::DOUBLE);+		} else if (format == "d:38,0") { // decimal128+			return_types.push_back(LogicalType::HUGEINT);+		} else if (format == "u") {+			return_types.push_back(LogicalType::VARCHAR);+		} else if (format == "tsn:") {+			return_types.push_back(LogicalType::TIMESTAMP);+		} else {+			throw NotImplementedException("Unsupported Arrow type %s", format);+		}+		auto name = string(schema.name);+		if (name.empty()) {+			name = string("v") + to_string(col_idx);+		}+		names.push_back(name);+	}+	data.ReleaseSchema();+	return move(res);+}++static void arrow_scan_function(ClientContext &context, vector<Value> &input, DataChunk &output,+                                FunctionData *dataptr) {+	auto &data = *((ArrowScanFunctionData *)dataptr);++	if (!data.stream->release) { // no more chunks+		return;+	}++	// have we run out of data on the current chunk? move to next one+	if (data.chunk_offset >= (idx_t)data.current_chunk_root.length) {+		data.chunk_offset = 0;+		data.ReleaseArray();+		if (data.stream->get_next(data.stream, &data.current_chunk_root)) {+			throw InvalidInputException("arrow_scan: get_next failed(): %s",+			                            string(data.stream->get_last_error(data.stream)));+		}+	}++	// have we run out of chunks? we done+	if (!data.current_chunk_root.release) {+		data.stream->release(data.stream);+		return;+	}++	if ((idx_t)data.current_chunk_root.n_children != output.column_count()) {+		throw InvalidInputException("arrow_scan: array column count mismatch");+	}++	output.SetCardinality(+	    std::min((int64_t)STANDARD_VECTOR_SIZE, (int64_t)(data.current_chunk_root.length - data.chunk_offset)));++	for (idx_t col_idx = 0; col_idx < output.column_count(); col_idx++) {+		auto &array = *data.current_chunk_root.children[col_idx];+		if (!array.release) {+			throw InvalidInputException("arrow_scan: released array passed");+		}+		if (array.length != data.current_chunk_root.length) {+			throw InvalidInputException("arrow_scan: array length mismatch");+		}+		if (array.dictionary) {+			throw NotImplementedException("arrow_scan: dictionary vectors not supported yet");+		}+		if (array.null_count != 0 && array.buffers[0]) {+			auto &nullmask = FlatVector::Nullmask(output.data[col_idx]);++			auto bit_offset = data.chunk_offset + array.offset;+			auto n_bitmask_bytes = (output.size() + 8 - 1) / 8;++			if (bit_offset % 8 == 0) {+				// just memcpy nullmask+				memcpy(&nullmask, (uint8_t *)array.buffers[0] + bit_offset / 8, n_bitmask_bytes);+			} else {+				// need to re-align nullmask :/+				bitset<STANDARD_VECTOR_SIZE + 8> temp_nullmask;+				memcpy(&temp_nullmask, (uint8_t *)array.buffers[0] + bit_offset / 8, n_bitmask_bytes + 1);++				temp_nullmask >>= (bit_offset % 8); // why this has to be a right shift is a mystery to me

 Status ReadDictionary(const Buffer& metadata, DictionaryMemo* dictionary_memo,     return Status::Invalid("Dictionary record batch must only contain one field");   }   auto dictionary = batch->column(0);-  // Validate the dictionary for safe delta concatenation-  RETURN_NOT_OK(dictionary->Validate());+  // Validate the dictionary for safe delta concatenation.  Full validation+  // is necessary for certain types (for example nested dictionaries).+  // XXX: instead only validate when a concatenation is actually done?+  RETURN_NOT_OK(dictionary->ValidateFull());

+from abc import ABC, abstractmethod+from collections.abc import Mapping+from typing import Any, Iterable, Sequence++# ----------------------------------------------------------------------+# A simple data type class hierarchy for illustration+++class DataType(ABC):+    """+    A metadata object representing the logical value type of a cell in a data+    frame column. This metadata does not guarantee an specific underlying data+    representation+    """+    def __eq__(self, other: 'DataType'):+        return self.equals(other)++    def __str__(self):+        return self.to_string()++    def __repr__(self):+        return str(self)++    @abstractmethod+    def to_string(self) -> str:+        """+        Return human-readable representation of the data type+        """++    @abstractmethod+    def equals(self, other: 'DataType') -> bool:+        """+        Return true if other DataType contains the same metadata as this+        DataType+        """+        pass+++class PrimitiveType(DataType):++    def equals(self, other: DataType) -> bool:+        return type(self) == type(other)+++class NullType(PrimitiveType):+    """+    A data type whose values are always null+    """+    def to_string(self):+        return "null"+++class Boolean(PrimitiveType):++    def to_string(self):+        return "bool"+++class NumberType(PrimitiveType):+    pass+++class IntegerType(NumberType):+    pass+++class SignedIntegerType(IntegerType):+    pass+++class Int8(SignedIntegerType):++    def to_string(self):+        return "int8"+++class Int16(SignedIntegerType):++    def to_string(self):+        return "int16"+++class Int32(SignedIntegerType):++    def to_string(self):+        return "int32"+++class Int64(SignedIntegerType):++    def to_string(self):+        return "int64"+++class Binary(PrimitiveType):+    """+    A variable-size binary (bytes) value+    """+    def to_string(self):+        return "binary"+++class String(PrimitiveType):+    """+    A UTF8-encoded string value+    """+    def to_string(self):+        return "string"+++class Object(PrimitiveType):+    """+    Any PyObject value+    """+    def to_string(self):+        return "object"+++class Categorical(DataType):+    """+    A categorical value is an ordinal (integer) value that references a+    sequence of category values of an arbitrary data type+    """++    def __init__(self, index_type: IntegerType, category_type: DataType,+                 ordered: bool = False):