upgrade to 0.8.0

aliyun · Dec 25, 2018 · f409b26 · f409b26
1 parent fe7bc61
commit f409b26
Show file tree

Hide file tree

Showing 37 changed files with 1,411 additions and 360 deletions.
diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -55,4 +55,5 @@ PyODPS 的相关依赖会自动安装。
    interactive
    options
    faq
+   mars-int
    api
diff --git a/docs/source/locale/en/LC_MESSAGES/api-df.po b/docs/source/locale/en/LC_MESSAGES/api-df.po
@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PyODPS 0.7.16\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2018-11-08 12:41+0800\n"
+"POT-Creation-Date: 2018-12-05 22:57+0800\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <[email protected]>\n"
@@ -890,20 +890,33 @@ msgstr ""
 msgid "instance priority, 9 as default"
 msgstr ""
 
-#: odps.df.CollectionExpr.execute:5 odps.df.RandomScalar.execute:5
-#: odps.df.Scalar.execute:5 odps.df.SequenceExpr.execute:5
+#: odps.df.CollectionExpr.execute:5 odps.df.CollectionExpr.persist:19
+#: odps.df.RandomScalar.execute:5 odps.df.RandomScalar.persist:19
+#: odps.df.Scalar.execute:5 odps.df.Scalar.persist:19
+#: odps.df.SequenceExpr.execute:5 odps.df.SequenceExpr.persist:19
 #: odps.df.expr.groupby.GroupBy.execute:5
+#: odps.df.expr.groupby.GroupBy.persist:19
 #: odps.df.expr.groupby.Int64SequenceGroupBy.execute:5
-#: odps.df.expr.groupby.SequenceGroupBy.execute:5 of
-msgid "execution result"
+#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:19
+#: odps.df.expr.groupby.SequenceGroupBy.execute:5
+#: odps.df.expr.groupby.SequenceGroupBy.persist:19 of
+msgid "cluster to run this instance"
 msgstr ""
 
-#: odps.df.CollectionExpr.execute:6 odps.df.CollectionExpr.head:5
-#: odps.df.CollectionExpr.tail:5 odps.df.RandomScalar.execute:6
+#: odps.df.CollectionExpr.execute:6 odps.df.RandomScalar.execute:6
 #: odps.df.Scalar.execute:6 odps.df.SequenceExpr.execute:6
 #: odps.df.expr.groupby.GroupBy.execute:6
 #: odps.df.expr.groupby.Int64SequenceGroupBy.execute:6
 #: odps.df.expr.groupby.SequenceGroupBy.execute:6 of
+msgid "execution result"
+msgstr ""
+
+#: odps.df.CollectionExpr.execute:7 odps.df.CollectionExpr.head:5
+#: odps.df.CollectionExpr.tail:5 odps.df.RandomScalar.execute:7
+#: odps.df.Scalar.execute:7 odps.df.SequenceExpr.execute:7
+#: odps.df.expr.groupby.GroupBy.execute:7
+#: odps.df.expr.groupby.Int64SequenceGroupBy.execute:7
+#: odps.df.expr.groupby.SequenceGroupBy.execute:7 of
 msgid ":class:`odps.df.backends.frame.ResultFrame`"
 msgstr ""
 
@@ -1364,59 +1377,59 @@ msgstr ""
 msgid "project name, if not provided, will be the default project"
 msgstr ""
 
-#: odps.df.CollectionExpr.persist:19 odps.df.RandomScalar.persist:19
-#: odps.df.Scalar.persist:19 odps.df.SequenceExpr.persist:19
-#: odps.df.expr.groupby.GroupBy.persist:19
-#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:19
-#: odps.df.expr.groupby.SequenceGroupBy.persist:19 of
+#: odps.df.CollectionExpr.persist:20 odps.df.RandomScalar.persist:20
+#: odps.df.Scalar.persist:20 odps.df.SequenceExpr.persist:20
+#: odps.df.expr.groupby.GroupBy.persist:20
+#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:20
+#: odps.df.expr.groupby.SequenceGroupBy.persist:20 of
 msgid "overwrite the table, True as default"
 msgstr ""
 
-#: odps.df.CollectionExpr.persist:21 odps.df.RandomScalar.persist:21
-#: odps.df.Scalar.persist:21 odps.df.SequenceExpr.persist:21
-#: odps.df.expr.groupby.GroupBy.persist:21
-#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:21
-#: odps.df.expr.groupby.SequenceGroupBy.persist:21 of
+#: odps.df.CollectionExpr.persist:22 odps.df.RandomScalar.persist:22
+#: odps.df.Scalar.persist:22 odps.df.SequenceExpr.persist:22
+#: odps.df.expr.groupby.GroupBy.persist:22
+#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:22
+#: odps.df.expr.groupby.SequenceGroupBy.persist:22 of
 msgid "drop table if exists, False as default"
 msgstr ""
 
-#: odps.df.CollectionExpr.persist:23 odps.df.RandomScalar.persist:23
-#: odps.df.Scalar.persist:23 odps.df.SequenceExpr.persist:23
-#: odps.df.expr.groupby.GroupBy.persist:23
-#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:23
-#: odps.df.expr.groupby.SequenceGroupBy.persist:23 of
+#: odps.df.CollectionExpr.persist:24 odps.df.RandomScalar.persist:24
+#: odps.df.Scalar.persist:24 odps.df.SequenceExpr.persist:24
+#: odps.df.expr.groupby.GroupBy.persist:24
+#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:24
+#: odps.df.expr.groupby.SequenceGroupBy.persist:24 of
 msgid "create table first if not exits, True as default"
 msgstr ""
 
-#: odps.df.CollectionExpr.persist:25 odps.df.RandomScalar.persist:25
-#: odps.df.Scalar.persist:25 odps.df.SequenceExpr.persist:25
-#: odps.df.expr.groupby.GroupBy.persist:25
-#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:25
-#: odps.df.expr.groupby.SequenceGroupBy.persist:25 of
+#: odps.df.CollectionExpr.persist:26 odps.df.RandomScalar.persist:26
+#: odps.df.Scalar.persist:26 odps.df.SequenceExpr.persist:26
+#: odps.df.expr.groupby.GroupBy.persist:26
+#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:26
+#: odps.df.expr.groupby.SequenceGroupBy.persist:26 of
 msgid "drop partition if exists, False as default"
 msgstr ""
 
-#: odps.df.CollectionExpr.persist:27 odps.df.RandomScalar.persist:27
-#: odps.df.Scalar.persist:27 odps.df.SequenceExpr.persist:27
-#: odps.df.expr.groupby.GroupBy.persist:27
-#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:27
-#: odps.df.expr.groupby.SequenceGroupBy.persist:27 of
+#: odps.df.CollectionExpr.persist:28 odps.df.RandomScalar.persist:28
+#: odps.df.Scalar.persist:28 odps.df.SequenceExpr.persist:28
+#: odps.df.expr.groupby.GroupBy.persist:28
+#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:28
+#: odps.df.expr.groupby.SequenceGroupBy.persist:28 of
 msgid "create partition if not exists, None as default"
 msgstr ""
 
-#: odps.df.CollectionExpr.persist:29 odps.df.RandomScalar.persist:29
-#: odps.df.Scalar.persist:29 odps.df.SequenceExpr.persist:29
-#: odps.df.expr.groupby.GroupBy.persist:29
-#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:29
-#: odps.df.expr.groupby.SequenceGroupBy.persist:29 of
+#: odps.df.CollectionExpr.persist:30 odps.df.RandomScalar.persist:30
+#: odps.df.Scalar.persist:30 odps.df.SequenceExpr.persist:30
+#: odps.df.expr.groupby.GroupBy.persist:30
+#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:30
+#: odps.df.expr.groupby.SequenceGroupBy.persist:30 of
 msgid "cast all columns' types as the existed table, False as default"
 msgstr ""
 
-#: odps.df.CollectionExpr.persist:31 odps.df.RandomScalar.persist:31
-#: odps.df.Scalar.persist:31 odps.df.SequenceExpr.persist:31
-#: odps.df.expr.groupby.GroupBy.persist:31
-#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:31
-#: odps.df.expr.groupby.SequenceGroupBy.persist:31 of
+#: odps.df.CollectionExpr.persist:32 odps.df.RandomScalar.persist:32
+#: odps.df.Scalar.persist:32 odps.df.SequenceExpr.persist:32
+#: odps.df.expr.groupby.GroupBy.persist:32
+#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:32
+#: odps.df.expr.groupby.SequenceGroupBy.persist:32 of
 msgid ":class:`odps.df.DataFrame`"
 msgstr ""
 

diff --git a/docs/source/locale/en/LC_MESSAGES/options.po b/docs/source/locale/en/LC_MESSAGES/options.po
@@ -6,16 +6,16 @@
 #, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: PyODPS 0.7.16\n"
+"Project-Id-Version: PyODPS 0.8.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2018-06-21 09:15+0800\n"
+"POT-Creation-Date: 2018-12-05 11:38+0800\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <[email protected]>\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"Generated-By: Babel 2.5.3\n"
+"Generated-By: Babel 2.6.0\n"
 
 #: ../../source/options.rst:5
 msgid "配置选项"
@@ -79,12 +79,12 @@ msgid "ODPS Endpoint"
 msgstr "MaxCompute Endpoint"
 
 #: ../../source/options.rst:27 ../../source/options.rst:29
-#: ../../source/options.rst:31 ../../source/options.rst:38
-#: ../../source/options.rst:42 ../../source/options.rst:46
-#: ../../source/options.rst:60 ../../source/options.rst:62
-#: ../../source/options.rst:68 ../../source/options.rst:79
-#: ../../source/options.rst:81 ../../source/options.rst:98
-#: ../../source/options.rst:110
+#: ../../source/options.rst:31 ../../source/options.rst:35
+#: ../../source/options.rst:38 ../../source/options.rst:42
+#: ../../source/options.rst:46 ../../source/options.rst:60
+#: ../../source/options.rst:62 ../../source/options.rst:68
+#: ../../source/options.rst:79 ../../source/options.rst:81
+#: ../../source/options.rst:98 ../../source/options.rst:110
 msgid "None"
 msgstr ""
 
@@ -121,14 +121,11 @@ msgid "local_timezone"
 msgstr ""
 
 #: ../../source/options.rst:35
-msgid "使用的时区，True 表示本地时间，False 表示 UTC， 也可用 pytz 的时区"
-msgstr ""
-"Used time zone. True indicates local time, and False indicates UTC. The "
-"time zone of pytz can also be used."
-
-#: ../../source/options.rst:35 ../../source/options.rst:40
-msgid "1"
+msgid "使用的时区，None 表示不处理，True 表示本地时区， False 表示 UTC，也可用 pytz 的时区"
 msgstr ""
+"Used time zone. None indicates that PyODPS takes no actions, True "
+"indicates local time, and False indicates UTC. The time zone of pytz "
+"package can also be used."
 
 #: ../../source/options.rst:38
 msgid "lifecycle"
@@ -146,6 +143,10 @@ msgstr ""
 msgid "临时表生命周期"
 msgstr "Life cycles of temporary tables"
 
+#: ../../source/options.rst:40
+msgid "1"
+msgstr ""
+
 #: ../../source/options.rst:42
 msgid "biz_id"
 msgstr ""

diff --git a/docs/source/options.rst b/docs/source/options.rst
@@ -32,8 +32,8 @@ PyODPS 提供了一系列的配置选项，可通过 ``odps.options`` 获得，
 +------------------------+---------------------------------------------------+-------+
 |log_view_hours          | LogView 保持时间（小时）                          |24     |
 +------------------------+---------------------------------------------------+-------+
-|local_timezone          | 使用的时区，True 表示本地时间，False 表示 UTC，   |1      |
-|                        | 也可用 pytz 的时区                                |       |
+|local_timezone          | 使用的时区，None 表示不处理，True 表示本地时区，  |None   |
+|                        | False 表示 UTC，也可用 pytz 的时区                |       |
 +------------------------+---------------------------------------------------+-------+
 |lifecycle               | 所有表生命周期                                    |None   |
 +------------------------+---------------------------------------------------+-------+

diff --git a/odps/_version.py b/odps/_version.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-version_info = (0, 7, 20, 1)
+version_info = (0, 8, 0)
 _num_index = max(idx if isinstance(v, int) else 0
                  for idx, v in enumerate(version_info))
 __version__ = '.'.join(map(str, version_info[:_num_index + 1])) + \

diff --git a/odps/config.py b/odps/config.py
@@ -325,7 +325,8 @@ def validate(x):
 options.register_option('account', None)
 options.register_option('end_point', None)
 options.register_option('default_project', None)
-options.register_option('local_timezone', True)
+options.register_option('local_timezone', None)
+options.register_option('allow_antique_date', False)
 options.register_option('user_agent_pattern', '$pyodps_version $python_version $os_version')
 options.register_option('log_view_host', None)
 options.register_option('log_view_hours', 24, validator=is_integer)

diff --git a/odps/df/__init__.py b/odps/df/__init__.py
@@ -59,3 +59,10 @@ def inner(*args, **kwargs):
             locals()[k] = wrap(v)
 except ImportError:
     pass
+
+try:
+    from ..internal import df as internal_df
+
+    del internal_df
+except ImportError:
+    pass
diff --git a/odps/df/backends/engine.py b/odps/df/backends/engine.py
@@ -70,7 +70,7 @@ def get_default_engine(*exprs):
             odps = expr_odps
 
     if odps is None and options.account is not None and \
-                    options.end_point is not None and options.default_project is not None:
+            options.end_point is not None and options.default_project is not None:
         odps = ODPS._from_account(options.account, options.default_project,
                                   endpoint=options.end_point,
                                   tunnel_endpoint=options.tunnel.endpoint)

diff --git a/odps/df/backends/odpssql/engine.py b/odps/df/backends/odpssql/engine.py
@@ -129,10 +129,11 @@ def _reload_ui(self, group, instance, ui):
             return fetch_instance_group(group).instances.get(instance.id)
 
     def _run(self, sql, ui, progress_proportion=1, hints=None, priority=None,
-             group=None, libraries=None):
+             running_cluster=None, group=None, libraries=None):
         libraries = self._ctx.prepare_resources(self._get_libraries(libraries))
         self._ctx.create_udfs(libraries=libraries)
-        instance = self._odps.run_sql(sql, hints=hints, priority=priority, name='PyODPSDataFrameTask')
+        instance = self._odps.run_sql(sql, hints=hints, priority=priority, name='PyODPSDataFrameTask',
+                                      running_cluster=running_cluster)
 
         self._instances.append(instance.id)
         log('Instance ID: ' + instance.id)
@@ -287,7 +288,7 @@ def _join_sql(cls, sql):
 
     def _do_execute(self, expr_dag, expr, ui=None, progress_proportion=1,
                     lifecycle=None, head=None, tail=None,
-                    hints=None, priority=None, **kw):
+                    hints=None, priority=None, running_cluster=None, **kw):
         lifecycle = lifecycle or options.temp_lifecycle
         group = kw.get('group')
         libraries = kw.pop('libraries', None)
@@ -344,7 +345,7 @@ def _do_execute(self, expr_dag, expr, ui=None, progress_proportion=1,
         log(sql)
 
         instance = self._run(sql, ui, progress_proportion=progress_proportion*0.9,
-                             hints=hints, priority=priority,
+                             hints=hints, priority=priority, running_cluster=running_cluster,
                              group=group, libraries=libraries)
 
         self._ctx.close()  # clear udfs and resources generated
@@ -433,8 +434,8 @@ def _fetch(self, expr, src_expr, instance, ui, progress_proportion=1,
 
     def _do_persist(self, expr_dag, expr, name, partitions=None, partition=None, project=None, ui=None,
                     progress_proportion=1, lifecycle=None, hints=None, priority=None,
-                    overwrite=True, drop_table=False, create_table=True, drop_partition=False,
-                    create_partition=None, cast=False, **kw):
+                    running_cluster=None, overwrite=True, drop_table=False, create_table=True,
+                    drop_partition=False, create_partition=None, cast=False, **kw):
         group = kw.get('group')
         libraries = kw.pop('libraries', None)
 
@@ -567,7 +568,8 @@ def _do_persist(self, expr_dag, expr, name, partitions=None, partition=None, pro
         log(sql)
 
         instance = self._run(sql, ui, progress_proportion=progress_proportion,
-                             hints=hints, priority=priority, group=group, libraries=libraries)
+                             hints=hints, priority=priority, running_cluster=running_cluster,
+                             group=group, libraries=libraries)
         self._ctx.close()  # clear udfs and resources generated
         t = self._odps.get_table(name, project=project)
         if should_cache and not is_source_collection(src_expr):

diff --git a/odps/df/expr/expressions.py b/odps/df/expr/expressions.py
@@ -176,6 +176,7 @@ def execute(self, **kwargs):
         :type hints: dict
         :param priority: instance priority, 9 as default
         :type priority: int
+        :param running_cluster: cluster to run this instance
         :return: execution result
         :rtype: :class:`odps.df.backends.frame.ResultFrame`
         """
@@ -223,6 +224,7 @@ def persist(self, name, partitions=None, partition=None, lifecycle=None, project
         :type hints: dict
         :param priority: instance priority, 9 as default
         :type priority: int
+        :param running_cluster: cluster to run this instance
         :param overwrite: overwrite the table, True as default
         :type overwrite: bool
         :param drop_table: drop table if exists, False as default

diff --git a/odps/lib/tzlocal/LICENSE.txt b/odps/lib/tzlocal/LICENSE.txt
@@ -0,0 +1,19 @@
+Copyright 2011-2017 Lennart Regebro
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/odps/lib/tzlocal/__init__.py b/odps/lib/tzlocal/__init__.py
@@ -0,0 +1,5 @@
+import sys
+if sys.platform == 'win32':
+    from .win32 import get_localzone, reload_localzone
+else:
+    from .unix import get_localzone, reload_localzone
-Original file line number
+Diff line change
@@ Expand Up / @@ -55,4 +55,5 @@ PyODPS 的相关依赖会自动安装。 @@
        interactive
        options
        faq
+       mars-int
        api