1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
|
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# distutils: language = c++
from pyarrow.includes.common cimport *
from pyarrow.includes.libarrow cimport *
cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil:
ctypedef enum CFileType "arrow::fs::FileType":
CFileType_NotFound "arrow::fs::FileType::NotFound"
CFileType_Unknown "arrow::fs::FileType::Unknown"
CFileType_File "arrow::fs::FileType::File"
CFileType_Directory "arrow::fs::FileType::Directory"
cdef cppclass CFileInfo "arrow::fs::FileInfo":
CFileInfo()
CFileInfo(CFileInfo&&)
CFileInfo& operator=(CFileInfo&&)
CFileInfo(const CFileInfo&)
CFileInfo& operator=(const CFileInfo&)
CFileType type()
void set_type(CFileType type)
c_string path()
void set_path(const c_string& path)
c_string base_name()
int64_t size()
void set_size(int64_t size)
c_string extension()
CTimePoint mtime()
void set_mtime(CTimePoint mtime)
cdef cppclass CFileSelector "arrow::fs::FileSelector":
CFileSelector()
c_string base_dir
c_bool allow_not_found
c_bool recursive
cdef cppclass CFileLocator "arrow::fs::FileLocator":
shared_ptr[CFileSystem] filesystem
c_string path
cdef cppclass CFileSystem "arrow::fs::FileSystem":
shared_ptr[CFileSystem] shared_from_this()
c_string type_name() const
CResult[c_string] NormalizePath(c_string path)
CResult[CFileInfo] GetFileInfo(const c_string& path)
CResult[vector[CFileInfo]] GetFileInfo(
const vector[c_string]& paths)
CResult[vector[CFileInfo]] GetFileInfo(const CFileSelector& select)
CStatus CreateDir(const c_string& path, c_bool recursive)
CStatus DeleteDir(const c_string& path)
CStatus DeleteDirContents(const c_string& path)
CStatus DeleteRootDirContents()
CStatus DeleteFile(const c_string& path)
CStatus DeleteFiles(const vector[c_string]& paths)
CStatus Move(const c_string& src, const c_string& dest)
CStatus CopyFile(const c_string& src, const c_string& dest)
CResult[shared_ptr[CInputStream]] OpenInputStream(
const c_string& path)
CResult[shared_ptr[CRandomAccessFile]] OpenInputFile(
const c_string& path)
CResult[shared_ptr[COutputStream]] OpenOutputStream(
const c_string& path, const shared_ptr[const CKeyValueMetadata]&)
CResult[shared_ptr[COutputStream]] OpenAppendStream(
const c_string& path, const shared_ptr[const CKeyValueMetadata]&)
c_bool Equals(const CFileSystem& other)
c_bool Equals(shared_ptr[CFileSystem] other)
CResult[shared_ptr[CFileSystem]] CFileSystemFromUri \
"arrow::fs::FileSystemFromUri"(const c_string& uri, c_string* out_path)
CResult[shared_ptr[CFileSystem]] CFileSystemFromUriOrPath \
"arrow::fs::FileSystemFromUriOrPath"(const c_string& uri,
c_string* out_path)
cdef cppclass CFileSystemGlobalOptions \
"arrow::fs::FileSystemGlobalOptions":
c_string tls_ca_file_path
c_string tls_ca_dir_path
CStatus CFileSystemsInitialize "arrow::fs::Initialize" \
(const CFileSystemGlobalOptions& options)
cdef cppclass CLocalFileSystemOptions "arrow::fs::LocalFileSystemOptions":
c_bool use_mmap
@staticmethod
CLocalFileSystemOptions Defaults()
c_bool Equals(const CLocalFileSystemOptions& other)
cdef cppclass CLocalFileSystem "arrow::fs::LocalFileSystem"(CFileSystem):
CLocalFileSystem()
CLocalFileSystem(CLocalFileSystemOptions)
CLocalFileSystemOptions options()
cdef cppclass CSubTreeFileSystem \
"arrow::fs::SubTreeFileSystem"(CFileSystem):
CSubTreeFileSystem(const c_string& base_path,
shared_ptr[CFileSystem] base_fs)
c_string base_path()
shared_ptr[CFileSystem] base_fs()
ctypedef enum CS3LogLevel "arrow::fs::S3LogLevel":
CS3LogLevel_Off "arrow::fs::S3LogLevel::Off"
CS3LogLevel_Fatal "arrow::fs::S3LogLevel::Fatal"
CS3LogLevel_Error "arrow::fs::S3LogLevel::Error"
CS3LogLevel_Warn "arrow::fs::S3LogLevel::Warn"
CS3LogLevel_Info "arrow::fs::S3LogLevel::Info"
CS3LogLevel_Debug "arrow::fs::S3LogLevel::Debug"
CS3LogLevel_Trace "arrow::fs::S3LogLevel::Trace"
cdef struct CS3GlobalOptions "arrow::fs::S3GlobalOptions":
CS3LogLevel log_level
cdef cppclass CS3ProxyOptions "arrow::fs::S3ProxyOptions":
c_string scheme
c_string host
int port
c_string username
c_string password
c_bool Equals(const CS3ProxyOptions& other)
@staticmethod
CResult[CS3ProxyOptions] FromUriString "FromUri"(
const c_string& uri_string)
ctypedef enum CS3CredentialsKind "arrow::fs::S3CredentialsKind":
CS3CredentialsKind_Anonymous "arrow::fs::S3CredentialsKind::Anonymous"
CS3CredentialsKind_Default "arrow::fs::S3CredentialsKind::Default"
CS3CredentialsKind_Explicit "arrow::fs::S3CredentialsKind::Explicit"
CS3CredentialsKind_Role "arrow::fs::S3CredentialsKind::Role"
CS3CredentialsKind_WebIdentity \
"arrow::fs::S3CredentialsKind::WebIdentity"
cdef cppclass CS3Options "arrow::fs::S3Options":
c_string region
c_string endpoint_override
c_string scheme
c_bool background_writes
shared_ptr[const CKeyValueMetadata] default_metadata
c_string role_arn
c_string session_name
c_string external_id
int load_frequency
CS3ProxyOptions proxy_options
CS3CredentialsKind credentials_kind
void ConfigureDefaultCredentials()
void ConfigureAccessKey(const c_string& access_key,
const c_string& secret_key,
const c_string& session_token)
c_string GetAccessKey()
c_string GetSecretKey()
c_string GetSessionToken()
c_bool Equals(const CS3Options& other)
@staticmethod
CS3Options Defaults()
@staticmethod
CS3Options Anonymous()
@staticmethod
CS3Options FromAccessKey(const c_string& access_key,
const c_string& secret_key,
const c_string& session_token)
@staticmethod
CS3Options FromAssumeRole(const c_string& role_arn,
const c_string& session_name,
const c_string& external_id,
const int load_frequency)
cdef cppclass CS3FileSystem "arrow::fs::S3FileSystem"(CFileSystem):
@staticmethod
CResult[shared_ptr[CS3FileSystem]] Make(const CS3Options& options)
CS3Options options()
c_string region()
cdef CStatus CInitializeS3 "arrow::fs::InitializeS3"(
const CS3GlobalOptions& options)
cdef CStatus CFinalizeS3 "arrow::fs::FinalizeS3"()
cdef cppclass CHdfsOptions "arrow::fs::HdfsOptions":
HdfsConnectionConfig connection_config
int32_t buffer_size
int16_t replication
int64_t default_block_size
@staticmethod
CResult[CHdfsOptions] FromUriString "FromUri"(
const c_string& uri_string)
void ConfigureEndPoint(c_string host, int port)
void ConfigureDriver(c_bool use_hdfs3)
void ConfigureReplication(int16_t replication)
void ConfigureUser(c_string user_name)
void ConfigureBufferSize(int32_t buffer_size)
void ConfigureBlockSize(int64_t default_block_size)
void ConfigureKerberosTicketCachePath(c_string path)
void ConfigureExtraConf(c_string key, c_string value)
cdef cppclass CHadoopFileSystem "arrow::fs::HadoopFileSystem"(CFileSystem):
@staticmethod
CResult[shared_ptr[CHadoopFileSystem]] Make(
const CHdfsOptions& options)
CHdfsOptions options()
cdef cppclass CMockFileSystem "arrow::fs::internal::MockFileSystem"(
CFileSystem):
CMockFileSystem(CTimePoint current_time)
CStatus CCopyFiles "arrow::fs::CopyFiles"(
const vector[CFileLocator]& sources,
const vector[CFileLocator]& destinations,
const CIOContext& io_context,
int64_t chunk_size, c_bool use_threads)
CStatus CCopyFilesWithSelector "arrow::fs::CopyFiles"(
const shared_ptr[CFileSystem]& source_fs,
const CFileSelector& source_sel,
const shared_ptr[CFileSystem]& destination_fs,
const c_string& destination_base_dir,
const CIOContext& io_context,
int64_t chunk_size, c_bool use_threads)
# Callbacks for implementing Python filesystems
# Use typedef to emulate syntax for std::function<void(..)>
ctypedef void CallbackGetTypeName(object, c_string*)
ctypedef c_bool CallbackEquals(object, const CFileSystem&)
ctypedef void CallbackGetFileInfo(object, const c_string&, CFileInfo*)
ctypedef void CallbackGetFileInfoVector(object, const vector[c_string]&,
vector[CFileInfo]*)
ctypedef void CallbackGetFileInfoSelector(object, const CFileSelector&,
vector[CFileInfo]*)
ctypedef void CallbackCreateDir(object, const c_string&, c_bool)
ctypedef void CallbackDeleteDir(object, const c_string&)
ctypedef void CallbackDeleteDirContents(object, const c_string&)
ctypedef void CallbackDeleteRootDirContents(object)
ctypedef void CallbackDeleteFile(object, const c_string&)
ctypedef void CallbackMove(object, const c_string&, const c_string&)
ctypedef void CallbackCopyFile(object, const c_string&, const c_string&)
ctypedef void CallbackOpenInputStream(object, const c_string&,
shared_ptr[CInputStream]*)
ctypedef void CallbackOpenInputFile(object, const c_string&,
shared_ptr[CRandomAccessFile]*)
ctypedef void CallbackOpenOutputStream(
object, const c_string&, const shared_ptr[const CKeyValueMetadata]&,
shared_ptr[COutputStream]*)
ctypedef void CallbackNormalizePath(object, const c_string&, c_string*)
cdef extern from "arrow/python/filesystem.h" namespace "arrow::py::fs" nogil:
cdef cppclass CPyFileSystemVtable "arrow::py::fs::PyFileSystemVtable":
PyFileSystemVtable()
function[CallbackGetTypeName] get_type_name
function[CallbackEquals] equals
function[CallbackGetFileInfo] get_file_info
function[CallbackGetFileInfoVector] get_file_info_vector
function[CallbackGetFileInfoSelector] get_file_info_selector
function[CallbackCreateDir] create_dir
function[CallbackDeleteDir] delete_dir
function[CallbackDeleteDirContents] delete_dir_contents
function[CallbackDeleteRootDirContents] delete_root_dir_contents
function[CallbackDeleteFile] delete_file
function[CallbackMove] move
function[CallbackCopyFile] copy_file
function[CallbackOpenInputStream] open_input_stream
function[CallbackOpenInputFile] open_input_file
function[CallbackOpenOutputStream] open_output_stream
function[CallbackOpenOutputStream] open_append_stream
function[CallbackNormalizePath] normalize_path
cdef cppclass CPyFileSystem "arrow::py::fs::PyFileSystem":
@staticmethod
shared_ptr[CPyFileSystem] Make(object handler,
CPyFileSystemVtable vtable)
PyObject* handler()
|
