Merge pull request #117 from aliyun/upgrade-to-0.9.0

Upgrade to 0.9.0

cupid/__init__.py

@@ -0,0 +1,19 @@
+# Copyright 1999-2017 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .session import CupidSession
+from .config import options  # no op
+from .runtime import context
+
+from ._version import version_info, __version__

cupid/_version.py

@@ -0,0 +1,15 @@
+# Copyright 1999-2017 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from odps._version import __version__, version_info

cupid/config.py

@@ -0,0 +1,35 @@
+# Copyright 1999-2017 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from odps import options
+
+
+options.register_option('cupid.major_task_version', 'cupid_v2')
+options.register_option('cupid.wait_am_start_time', 600)
+options.register_option('cupid.use_bearer_token', None)
+options.register_option('cupid.settings', None)
+options.register_option('cupid.mp_buffer_size', 1024 * 64)
+
+options.register_option('cupid.proxy_endpoint', 'open.maxcompute.aliyun.com')
+options.register_option('cupid.worker.virtual_resource', None)
+options.register_option('cupid.master_type', 'kubernetes')
+options.register_option('cupid.application_type', 'mars')
+options.register_option('cupid.running_engine_type', None)
+options.register_option('cupid.job_duration_hours', 25920)
+options.register_option('cupid.channel_init_timeout_seconds', 120)
+options.register_option('cupid.kube.master_mode', 'cupid')
+
+
+# mars app config
+options.register_option('cupid.mars_image', None)

cupid/errors.py

@@ -0,0 +1,69 @@
+# Copyright 1999-2017 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class SubprocessStreamEOFError(IOError):
+    pass
+
+
+class CupidError(RuntimeError):
+    pass
+
+
+class InstanceRecycledError(CupidError):
+    pass
+
+
+class CupidUserError(CupidError):
+    pass
+
+
+class CupidCppError(CupidError):
+    cpp_error_type = None
+
+    def __new__(cls, err_type, err_msg):
+        if cls is not CupidCppError:
+            return CupidError.__new__(cls)
+        if err_type in _cpp_errors:
+            return CupidError.__new__(_cpp_errors[err_type])
+        else:
+            return CupidError.__new__(cls)
+
+    def __init__(self, err_type, err_msg):
+        self._err_type = err_type
+        self._err_msg = err_msg
+
+    def __str__(self):
+        return '%s: %s' % (self._err_type, self._err_msg)
+
+
+class CupidReplyTimeoutError(CupidCppError):
+    cpp_error_type = 'ChannelTimeOutException'
+
+
+class CupidChannelReplyError(CupidCppError):
+    cpp_error_type = 'ChannelReplyException'
+
+
+class CupidClientClosedError(CupidCppError):
+    cpp_error_type = 'ChannelClientClosedException'
+
+
+_cpp_errors = dict()
+for _ex in globals().copy().values():
+    if not isinstance(_ex, type) or not issubclass(_ex, CupidCppError):
+        continue
+    if getattr(_ex, 'cpp_error_type', None) is None:
+        continue
+    _cpp_errors[_ex.cpp_error_type] = _ex

cupid/io/__init__.py

@@ -0,0 +1,16 @@
+# Copyright 1999-2017 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .table import TableSplit, CupidTableDownloadSession, CupidTableUploadSession
+from .kvstore import CupidKVStore

cupid/io/kvstore.py

@@ -0,0 +1,56 @@
+# Copyright 1999-2017 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from odps.compat import six
+
+from ..rpc import SandboxRpcChannel, CupidRpcController
+from ..proto import kv_store_service_pb2 as kv_pb
+from ..errors import CupidError
+
+
+class CupidKVStore(object):
+    def __init__(self):
+        self.channel = SandboxRpcChannel()
+        self.stub = kv_pb.KVStoreService_Stub(self.channel)
+
+    def __setitem__(self, key, value):
+        if isinstance(value, six.text_type):
+            value = value.encode()
+
+        controller = CupidRpcController()
+        req = kv_pb.PutRequest(key=key, value=value)
+
+        self.stub.Put(controller, req, None)
+        if controller.Failed():
+            raise CupidError(controller.ErrorText())
+
+    def __getitem__(self, item):
+        controller = CupidRpcController()
+
+        req = kv_pb.GetRequest(value=item)
+        resp = self.stub.Get(controller, req, None)
+
+        if controller.Failed():
+            err_text = controller.ErrorText()
+            if 'PANGU_FILE_NOT_FOUND' in err_text:
+                raise KeyError(err_text)
+            else:
+                raise CupidError(err_text)
+        return resp.value
+
+    def get(self, item, default=None):
+        try:
+            return self[item]
+        except KeyError:
+            return default

cupid/io/table/__init__.py

@@ -0,0 +1,21 @@
+# Copyright 1999-2017 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .core import CupidTableUploadSession, CupidTableDownloadSession, TableSplit
+from .record import CupidRecordReader, CupidRecordWriter
+
+try:
+    from .pd import CupidPandasReader, CupidPandasWriter
+except ImportError:
+    pass